From bfd4a614590ce6e753ebe367c9e71df61de8eb71 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 May 2026 17:17:06 +0800 Subject: [PATCH 001/137] fix: avoid duplicate bulk queue consumers Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../elastic/bulk_indexing/bulk_indexing.go | 110 +++++++++++------- .../bulk_indexing/bulk_indexing_test.go | 27 +++++ 2 files changed, 95 insertions(+), 42 deletions(-) diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index f7c1f118a..07c0e7031 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -433,6 +433,10 @@ func (processor *BulkIndexingProcessor) HandleQueueConfig(v *queue.QueueConfig, } func (processor *BulkIndexingProcessor) NewBulkWorker(parentContext *pipeline.Context, qConfig *queue.QueueConfig, preferedHost string) { + if global.Env().IsDebug { + // current time for monitoring and log + log.Debugf("starting bulk worker for queue: %v, host: %v at time: %v", qConfig.Name, preferedHost, time.Now().Format(time.RFC3339)) + } bulkSizeInByte := processor.config.BulkConfig.GetBulkSizeInBytes() //check slice for sliceID := 0; sliceID < processor.config.NumOfSlices; sliceID++ { @@ -460,50 +464,71 @@ func (processor *BulkIndexingProcessor) NewBulkWorker(parentContext *pipeline.Co return } - processor.Lock() - v2, exists := processor.inFlightQueueConfigs.Load(key) - if exists { + var workerID = util.GetUUID() + v2, reserved := processor.reserveInFlightQueue(key, workerID) + if !reserved { if global.Env().IsDebug { log.Tracef("[%v], queue [%v], slice_id:%v has more then one consumer, key:%v,v:%v", preferedHost, qConfig.ID, sliceID, key, v2) } - processor.Unlock() continue - } else { - var workerID = util.GetUUID() - log.Debugf("starting worker:[%v], queue:[%v], slice_id:%v, host:[%v]", workerID, qConfig.Name, sliceID, preferedHost) - - ctx1 := &pipeline.Context{} - ctx1.Set("key", key) - ctx1.Set("workerID", workerID) - ctx1.Set("sliceID", sliceID) - ctx1.Set("numOfSlices", processor.config.NumOfSlices) - ctx1.Set("tag", preferedHost) - ctx1.Set("qConfig", qConfig) - ctx1.Set("host", preferedHost) - ctx1.Set("bulkSizeInByte", bulkSizeInByte) - err := processor.pool.Submit(&pipeline.Task{ - Handler: func(ctx *pipeline.Context, v ...interface{}) { - key := ctx.MustGetString("key") - workerID := ctx.MustGetString("workerID") - host := ctx.MustGetString("host") - sliceID := ctx.MustGetInt("sliceID") - tag := ctx.MustGetString("tag") - numOfSlices := ctx.MustGetInt("numOfSlices") - bulkSizeInByte := ctx.MustGetInt("bulkSizeInByte") - qConfig := ctx.MustGet("qConfig").(*queue.QueueConfig) - pCtx := v[0].(*pipeline.Context) - processor.NewSlicedBulkWorker(pCtx, key, workerID, sliceID, numOfSlices, tag, bulkSizeInByte, qConfig, host) - }, - Context: ctx1, - Params: []interface{}{parentContext}, // 也可以在创建任务时设置参数 - }) - processor.Unlock() - if err != nil { - panic(err) - } - processor.wg.Add(1) } + + log.Debugf("starting worker:[%v], queue:[%v], slice_id:%v, host:[%v]", workerID, qConfig.Name, sliceID, preferedHost) + + ctx1 := &pipeline.Context{} + ctx1.Set("key", key) + ctx1.Set("workerID", workerID) + ctx1.Set("sliceID", sliceID) + ctx1.Set("numOfSlices", processor.config.NumOfSlices) + ctx1.Set("tag", preferedHost) + ctx1.Set("qConfig", qConfig) + ctx1.Set("host", preferedHost) + ctx1.Set("bulkSizeInByte", bulkSizeInByte) + err := processor.pool.Submit(&pipeline.Task{ + Handler: func(ctx *pipeline.Context, v ...interface{}) { + key := ctx.MustGetString("key") + workerID := ctx.MustGetString("workerID") + host := ctx.MustGetString("host") + sliceID := ctx.MustGetInt("sliceID") + tag := ctx.MustGetString("tag") + numOfSlices := ctx.MustGetInt("numOfSlices") + bulkSizeInByte := ctx.MustGetInt("bulkSizeInByte") + qConfig := ctx.MustGet("qConfig").(*queue.QueueConfig) + pCtx := v[0].(*pipeline.Context) + processor.NewSlicedBulkWorker(pCtx, key, workerID, sliceID, numOfSlices, tag, bulkSizeInByte, qConfig, host) + }, + Context: ctx1, + Params: []interface{}{parentContext}, // 也可以在创建任务时设置参数 + }) + if err != nil { + processor.inFlightQueueConfigs.Delete(key) + processor.wg.Done() + panic(err) + } + } +} + +func (processor *BulkIndexingProcessor) reserveInFlightQueue(key, workerID string) (interface{}, bool) { + processor.Lock() + defer processor.Unlock() + + v, exists := processor.inFlightQueueConfigs.Load(key) + if exists { + return v, false + } + + processor.inFlightQueueConfigs.Store(key, workerID) + processor.wg.Add(1) + + return workerID, true +} + +func isIgnorableAcquireConsumerError(err error) bool { + if err == nil { + return false } + + return util.ContainStr(err.Error(), "already owning this topic") || util.ContainStr(err.Error(), "the consumer is in fighting list") } var xxHashPool = sync.Pool{ @@ -549,8 +574,6 @@ func (processor *BulkIndexingProcessor) getConsumerConfig(queueID, consumerName } func (processor *BulkIndexingProcessor) NewSlicedBulkWorker(ctx *pipeline.Context, key, workerID string, sliceID, maxSlices int, tag string, bulkSizeInByte int, qConfig *queue.QueueConfig, host string) { - processor.inFlightQueueConfigs.Store(key, workerID) - defer func() { if !global.Env().IsDebug { if r := recover(); r != nil { @@ -600,12 +623,15 @@ func (processor *BulkIndexingProcessor) NewSlicedBulkWorker(ctx *pipeline.Contex var consumerInstance queue.ConsumerAPI consumerInstance, err = queue.AcquireConsumer(qConfig, consumerConfig, workerID) if err != nil || consumerInstance == nil { - if util.ContainStr(err.Error(), "already owning this topic") { + if isIgnorableAcquireConsumerError(err) { if global.Env().IsDebug { - log.Warnf("other consumer already owning this topic, queue:%v-%v, slice_id:%v", qConfig.Name, qConfig.ID, sliceID) + log.Warnf("skip duplicate consumer acquisition, queue:%v-%v, slice_id:%v, err:%v", qConfig.Name, qConfig.ID, sliceID, err) } return } + if err == nil { + err = errors.New("failed to acquire queue consumer") + } panic(err) } diff --git a/plugins/elastic/bulk_indexing/bulk_indexing_test.go b/plugins/elastic/bulk_indexing/bulk_indexing_test.go index e37b0ffe0..45eff76ed 100644 --- a/plugins/elastic/bulk_indexing/bulk_indexing_test.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing_test.go @@ -28,6 +28,7 @@ package bulk_indexing import ( + stdErrors "errors" "github.com/OneOfOne/xxhash" "github.com/stretchr/testify/assert" "testing" @@ -84,3 +85,29 @@ func TestXXHash(t *testing.T) { } } + +func TestReserveInFlightQueue(t *testing.T) { + processor := &BulkIndexingProcessor{} + + current, reserved := processor.reserveInFlightQueue("queue-0", "worker-1") + assert.True(t, reserved) + assert.Equal(t, "worker-1", current) + + stored, exists := processor.inFlightQueueConfigs.Load("queue-0") + assert.True(t, exists) + assert.Equal(t, "worker-1", stored) + + current, reserved = processor.reserveInFlightQueue("queue-0", "worker-2") + assert.False(t, reserved) + assert.Equal(t, "worker-1", current) + + processor.inFlightQueueConfigs.Delete("queue-0") + processor.wg.Done() +} + +func TestIsIgnorableAcquireConsumerError(t *testing.T) { + assert.True(t, isIgnorableAcquireConsumerError(stdErrors.New("already owning this topic"))) + assert.True(t, isIgnorableAcquireConsumerError(stdErrors.New("the consumer is in fighting list"))) + assert.False(t, isIgnorableAcquireConsumerError(stdErrors.New("some other error"))) + assert.False(t, isIgnorableAcquireConsumerError(nil)) +} From 010000ca11c474bc4ebeceb532f5e13a502f7278 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 May 2026 17:29:23 +0800 Subject: [PATCH 002/137] fix: keep consumer conflicts visible Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../elastic/bulk_indexing/bulk_indexing.go | 25 ++++++++++++++++--- .../bulk_indexing/bulk_indexing_test.go | 14 ++++++++++- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index 07c0e7031..275661139 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -299,7 +299,7 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { } //if have depth and not in in flight if !processor.config.SkipEmptyQueue || queue.HasLag(v) { - _, ok := processor.inFlightQueueConfigs.Load(v.ID) + ok := processor.hasInFlightQueue(v.ID) if !ok { if global.Env().IsDebug { log.Tracef("detecting new queue: %v", v.Name) @@ -339,7 +339,7 @@ const queueHandleSingleton = "queue_handler_singleton" func (processor *BulkIndexingProcessor) HandleQueueConfig(v *queue.QueueConfig, parentContext *pipeline.Context) { //TODO, add config to enable/disable singleton, may have performance issue - ok, _ := locker.Hold(queueHandleSingleton, v.ID, global.Env().SystemConfig.NodeConfig.ID, 60*time.Second, true) + ok, _ := locker.Hold(queueHandleSingleton, v.ID, processor.id, 60*time.Second, true) if !ok { log.Debugf("failed to hold lock for queue:[%v], already hold by somewhere", v.ID) return @@ -523,12 +523,31 @@ func (processor *BulkIndexingProcessor) reserveInFlightQueue(key, workerID strin return workerID, true } +func (processor *BulkIndexingProcessor) hasInFlightQueue(queueID string) bool { + if _, ok := processor.inFlightQueueConfigs.Load(queueID); ok { + return true + } + + queuePrefix := fmt.Sprintf("%v-", queueID) + hasInFlight := false + processor.inFlightQueueConfigs.Range(func(key, value interface{}) bool { + keyStr, ok := key.(string) + if ok && strings.HasPrefix(keyStr, queuePrefix) { + hasInFlight = true + return false + } + return true + }) + + return hasInFlight +} + func isIgnorableAcquireConsumerError(err error) bool { if err == nil { return false } - return util.ContainStr(err.Error(), "already owning this topic") || util.ContainStr(err.Error(), "the consumer is in fighting list") + return util.ContainStr(err.Error(), "already owning this topic") } var xxHashPool = sync.Pool{ diff --git a/plugins/elastic/bulk_indexing/bulk_indexing_test.go b/plugins/elastic/bulk_indexing/bulk_indexing_test.go index 45eff76ed..142ba83bd 100644 --- a/plugins/elastic/bulk_indexing/bulk_indexing_test.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing_test.go @@ -105,9 +105,21 @@ func TestReserveInFlightQueue(t *testing.T) { processor.wg.Done() } +func TestHasInFlightQueue(t *testing.T) { + processor := &BulkIndexingProcessor{} + + assert.False(t, processor.hasInFlightQueue("queue-0")) + + processor.inFlightQueueConfigs.Store("queue-0-0", "worker-1") + assert.True(t, processor.hasInFlightQueue("queue-0")) + + processor.inFlightQueueConfigs.Delete("queue-0-0") + assert.False(t, processor.hasInFlightQueue("queue-0")) +} + func TestIsIgnorableAcquireConsumerError(t *testing.T) { assert.True(t, isIgnorableAcquireConsumerError(stdErrors.New("already owning this topic"))) - assert.True(t, isIgnorableAcquireConsumerError(stdErrors.New("the consumer is in fighting list"))) + assert.False(t, isIgnorableAcquireConsumerError(stdErrors.New("the consumer is in fighting list"))) assert.False(t, isIgnorableAcquireConsumerError(stdErrors.New("some other error"))) assert.False(t, isIgnorableAcquireConsumerError(nil)) } From 543b468fe56ba09c358c6003d395ca89a2d73868 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 May 2026 17:57:18 +0800 Subject: [PATCH 003/137] fix: rate limit bulk queue lock logs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- plugins/elastic/bulk_indexing/bulk_indexing.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index 275661139..afbfec761 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -341,7 +341,9 @@ func (processor *BulkIndexingProcessor) HandleQueueConfig(v *queue.QueueConfig, //TODO, add config to enable/disable singleton, may have performance issue ok, _ := locker.Hold(queueHandleSingleton, v.ID, processor.id, 60*time.Second, true) if !ok { - log.Debugf("failed to hold lock for queue:[%v], already hold by somewhere", v.ID) + if rate.GetRateLimiter("bulk_queue_lock", v.ID, 1, 1, 30*time.Second).Allow() { + log.Debugf("failed to hold lock for queue:[%v], already hold by somewhere", v.ID) + } return } From 660c135f78beb7f1a5eaa77219c0308030196aee Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 May 2026 18:02:56 +0800 Subject: [PATCH 004/137] fix: avoid local bulk queue owner races Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../elastic/bulk_indexing/bulk_indexing.go | 32 ++++++++++++++++++- .../bulk_indexing/bulk_indexing_test.go | 31 ++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index afbfec761..5c7a32c6d 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -78,6 +78,8 @@ type BulkIndexingProcessor struct { bulkBufferPool *elastic.BulkBufferPool } +var queueOwners sync.Map + type Config struct { NumOfSlices int `config:"num_of_slices"` Slices []int `config:"slices"` @@ -337,9 +339,16 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { const queueHandleSingleton = "queue_handler_singleton" func (processor *BulkIndexingProcessor) HandleQueueConfig(v *queue.QueueConfig, parentContext *pipeline.Context) { + if !processor.acquireQueueOwner(v.ID) { + if rate.GetRateLimiter("bulk_queue_owner", v.ID, 1, 1, 30*time.Second).Allow() { + log.Debugf("skip queue:[%v], already owned by another local bulk processor", v.ID) + } + return + } + defer processor.releaseQueueOwnerIfIdle(v.ID) //TODO, add config to enable/disable singleton, may have performance issue - ok, _ := locker.Hold(queueHandleSingleton, v.ID, processor.id, 60*time.Second, true) + ok, _ := locker.Hold(queueHandleSingleton, v.ID, global.Env().SystemConfig.NodeConfig.ID, 60*time.Second, true) if !ok { if rate.GetRateLimiter("bulk_queue_lock", v.ID, 1, 1, 30*time.Second).Allow() { log.Debugf("failed to hold lock for queue:[%v], already hold by somewhere", v.ID) @@ -544,6 +553,26 @@ func (processor *BulkIndexingProcessor) hasInFlightQueue(queueID string) bool { return hasInFlight } +func (processor *BulkIndexingProcessor) acquireQueueOwner(queueID string) bool { + owner, loaded := queueOwners.LoadOrStore(queueID, processor.id) + if !loaded { + return true + } + + return owner == processor.id +} + +func (processor *BulkIndexingProcessor) releaseQueueOwnerIfIdle(queueID string) { + if processor.hasInFlightQueue(queueID) { + return + } + + owner, ok := queueOwners.Load(queueID) + if ok && owner == processor.id { + queueOwners.Delete(queueID) + } +} + func isIgnorableAcquireConsumerError(err error) bool { if err == nil { return false @@ -615,6 +644,7 @@ func (processor *BulkIndexingProcessor) NewSlicedBulkWorker(ctx *pipeline.Contex } } processor.inFlightQueueConfigs.Delete(key) + processor.releaseQueueOwnerIfIdle(qConfig.ID) processor.wg.Done() if global.Env().IsDebug { log.Tracef("exit slice worker, worker:[%v], queue:%v, slice_id:%v, key:%v", workerID, qConfig.ID, sliceID, key) diff --git a/plugins/elastic/bulk_indexing/bulk_indexing_test.go b/plugins/elastic/bulk_indexing/bulk_indexing_test.go index 142ba83bd..aa43bd8f2 100644 --- a/plugins/elastic/bulk_indexing/bulk_indexing_test.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing_test.go @@ -31,6 +31,7 @@ import ( stdErrors "errors" "github.com/OneOfOne/xxhash" "github.com/stretchr/testify/assert" + "sync" "testing" ) @@ -117,6 +118,36 @@ func TestHasInFlightQueue(t *testing.T) { assert.False(t, processor.hasInFlightQueue("queue-0")) } +func TestAcquireQueueOwner(t *testing.T) { + queueOwners = sync.Map{} + + processor1 := &BulkIndexingProcessor{id: "processor-1"} + processor2 := &BulkIndexingProcessor{id: "processor-2"} + + assert.True(t, processor1.acquireQueueOwner("queue-0")) + assert.True(t, processor1.acquireQueueOwner("queue-0")) + assert.False(t, processor2.acquireQueueOwner("queue-0")) + + queueOwners = sync.Map{} +} + +func TestReleaseQueueOwnerIfIdle(t *testing.T) { + queueOwners = sync.Map{} + + processor := &BulkIndexingProcessor{id: "processor-1"} + assert.True(t, processor.acquireQueueOwner("queue-0")) + + processor.inFlightQueueConfigs.Store("queue-0-0", "worker-1") + processor.releaseQueueOwnerIfIdle("queue-0") + _, exists := queueOwners.Load("queue-0") + assert.True(t, exists) + + processor.inFlightQueueConfigs.Delete("queue-0-0") + processor.releaseQueueOwnerIfIdle("queue-0") + _, exists = queueOwners.Load("queue-0") + assert.False(t, exists) +} + func TestIsIgnorableAcquireConsumerError(t *testing.T) { assert.True(t, isIgnorableAcquireConsumerError(stdErrors.New("already owning this topic"))) assert.False(t, isIgnorableAcquireConsumerError(stdErrors.New("the consumer is in fighting list"))) From a1778168656bce71121e08f7192e11fe3924063b Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 May 2026 18:44:33 +0800 Subject: [PATCH 005/137] docs: add release note for bulk queue consumer fix Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/content.en/docs/release-notes/_index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index fd0801d93..f03337132 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -24,6 +24,7 @@ Information about release notes of INFINI Framework is provided here. - feat: add pluggable sink to host metrics collectors #288 ### 🐛 Bug fix +- fix: prevent duplicate bulk queue consumers during bulk indexing migrations #289 ### ✈️ Improvements - chore: API Handler Registration Improvements #283 - refactor: use PathUnescape to decode query param filter #249 From 0095e10a29cf307918b3ea38039a28e5ca752490 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 9 May 2026 17:46:56 +0800 Subject: [PATCH 006/137] chore: for migration locat test --- core/elastic/index.go | 43 +++++++++ core/elastic/index_test.go | 30 ++++++ core/env/env.go | 2 +- core/queue/consumer_config.go | 2 +- core/queue/queue_config.go | 2 +- modules/elastic/adapter/elasticsearch/v0.go | 14 +++ modules/elastic/metadata.go | 16 ++-- modules/elastic/module.go | 24 ++++- modules/elastic/schema.go | 96 +++++++++++++++++++ modules/elastic/schema_test.go | 16 ++++ modules/metrics/metrics.go | 4 +- modules/queue/disk_queue/cleanup.go | 2 +- modules/queue/disk_queue/compress.go | 2 +- modules/queue/disk_queue/consumer.go | 6 +- modules/queue/disk_queue/diskqueue.go | 31 +++++- modules/queue/disk_queue/diskqueue_test.go | 40 ++++++++ modules/queue/disk_queue/module.go | 33 ++++++- modules/queue/disk_queue/module_test.go | 32 +++++++ .../elastic/bulk_indexing/bulk_indexing.go | 28 ++++-- 19 files changed, 392 insertions(+), 31 deletions(-) create mode 100644 modules/queue/disk_queue/diskqueue_test.go create mode 100644 modules/queue/disk_queue/module_test.go diff --git a/core/elastic/index.go b/core/elastic/index.go index 6ca476b22..6816e9173 100755 --- a/core/elastic/index.go +++ b/core/elastic/index.go @@ -24,6 +24,7 @@ package elastic import ( + "bytes" "errors" "github.com/buger/jsonparser" "github.com/segmentio/encoding/json" @@ -235,6 +236,48 @@ type ErrorDetail struct { Reason string `json:"reason,omitempty"` } +func (d *ErrorDetail) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || bytes.Equal(data, []byte("null")) { + return nil + } + + if len(data) > 0 && data[0] == '"' { + return json.Unmarshal(data, &d.Reason) + } + + type alias ErrorDetail + var aux alias + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + *d = ErrorDetail(aux) + return nil +} + +func (d *ErrorDetail) Message() string { + if d == nil { + return "" + } + if d.Reason != "" { + return d.Reason + } + if len(d.RootCause) > 0 { + var reasons []string + for _, cause := range d.RootCause { + if cause.Reason != "" { + reasons = append(reasons, cause.Reason) + } else if cause.Type != "" { + reasons = append(reasons, cause.Type) + } + } + if len(reasons) > 0 { + return strings.Join(reasons, "; ") + } + } + return d.Type +} + type RootCause struct { Type string `json:"type,omitempty"` Reason string `json:"reason,omitempty"` diff --git a/core/elastic/index_test.go b/core/elastic/index_test.go index d36ae4d1c..af98fdc3a 100644 --- a/core/elastic/index_test.go +++ b/core/elastic/index_test.go @@ -25,6 +25,8 @@ package elastic import ( "testing" + + "github.com/segmentio/encoding/json" ) func TestIndexDocument_GetStringFieldFromSource(t *testing.T) { @@ -220,3 +222,31 @@ func TestIndexDocument_TryGetStringFieldFromSource(t *testing.T) { }) } } + +func TestErrorDetailUnmarshalJSONString(t *testing.T) { + var detail ErrorDetail + if err := json.Unmarshal([]byte(`"initializing"`), &detail); err != nil { + t.Fatalf("unexpected unmarshal error: %v", err) + } + + if detail.Reason != "initializing" { + t.Fatalf("unexpected reason: %q", detail.Reason) + } + if detail.Message() != "initializing" { + t.Fatalf("unexpected message: %q", detail.Message()) + } +} + +func TestErrorDetailUnmarshalJSONObject(t *testing.T) { + var detail ErrorDetail + if err := json.Unmarshal([]byte(`{"type":"search_phase_execution_exception","reason":"all shards failed"}`), &detail); err != nil { + t.Fatalf("unexpected unmarshal error: %v", err) + } + + if detail.Type != "search_phase_execution_exception" { + t.Fatalf("unexpected type: %q", detail.Type) + } + if detail.Message() != "all shards failed" { + t.Fatalf("unexpected message: %q", detail.Message()) + } +} diff --git a/core/env/env.go b/core/env/env.go index 45b61ff8e..84601e6b7 100755 --- a/core/env/env.go +++ b/core/env/env.go @@ -550,7 +550,7 @@ func ParseConfigSection(cfg *config.Config, configKey string, configInstance int // go-ucfg raises an error if the key does not exist, in which case // we should return and report that the configKey does not exist. if ucfgErr, ok := err.(ucfg.Error); ok && ucfgErr.Reason() == ucfg.ErrMissing { - log.Debugf("config key: %s not found", configKey) + log.Tracef("config key: %s not found", configKey) return false, nil } diff --git a/core/queue/consumer_config.go b/core/queue/consumer_config.go index 8572bcf47..2f0b3f328 100644 --- a/core/queue/consumer_config.go +++ b/core/queue/consumer_config.go @@ -172,7 +172,7 @@ func RemoveAllConsumers(qConfig *QueueConfig) (bool, error) { log.Error(err) return false, err } - log.Debugf("success delete all consumers for queue:%v", qConfig.ID) + log.Tracef("success delete all consumers for queue:%v", qConfig.ID) return true, nil } diff --git a/core/queue/queue_config.go b/core/queue/queue_config.go index 8d2671869..d03b2f64c 100644 --- a/core/queue/queue_config.go +++ b/core/queue/queue_config.go @@ -118,7 +118,7 @@ func RegisterConfig(cfg *QueueConfig) (preExists bool, err error) { cfg.Created = time.Now().String() - log.Debug("init new queue config:", cfg.ID, ",", cfg.Name) + log.Trace("init new queue config:", cfg.ID, ",", cfg.Name) addCfgToCache(cfg) diff --git a/modules/elastic/adapter/elasticsearch/v0.go b/modules/elastic/adapter/elasticsearch/v0.go index 11538d859..f75251e9c 100755 --- a/modules/elastic/adapter/elasticsearch/v0.go +++ b/modules/elastic/adapter/elasticsearch/v0.go @@ -491,6 +491,20 @@ func (c *ESAPIV0) Get(indexName, docType, id string) (*elastic.GetResponse, erro return esResp, err } + if resp.StatusCode >= 400 { + if esResp.Error != nil { + errType := esResp.Error.Type + errReason := esResp.Error.Message() + if errType != "" && errReason != "" { + return esResp, errors.Errorf("status:%d, type:%s, reason:%s", resp.StatusCode, errType, errReason) + } + if errReason != "" { + return esResp, errors.Errorf("status:%d, reason:%s", resp.StatusCode, errReason) + } + } + return esResp, errors.Errorf("status:%d", resp.StatusCode) + } + return esResp, nil } diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index 60c07bdd9..1c6cff6ce 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -697,10 +697,13 @@ func (module *ElasticModule) updateNodeInfo(meta *elastic.ElasticsearchMetadata, log.Trace("update node info") if !force && !meta.IsAvailable() { + stateChanged := false if !force { - setNodeUnknown(meta.Config.ID) + stateChanged = setNodeUnknown(meta.Config.ID) + } + if stateChanged || rate.GetRateLimiter("metadata_node_info_skip", meta.Config.ID, 1, 1, 10*time.Minute).Allow() { + log.Debugf("elasticsearch [%v] is not available, skip update node info", meta.Config.Name) } - log.Debugf("elasticsearch [%v] is not available, skip update node info", meta.Config.Name) return } @@ -808,17 +811,17 @@ func (module *ElasticModule) updateNodeInfo(meta *elastic.ElasticsearchMetadata, var saveNodeMetadataMutex = sync.Mutex{} var nodeAlreadyUnknown = map[string]bool{} -func setNodeUnknown(clusterID string) { +func setNodeUnknown(clusterID string) bool { kv.DeleteKey(elastic.KVElasticNodeMetadata, []byte(clusterID)) meta := elastic.GetMetadata(clusterID) if meta == nil { - return + return false } if meta.Config.Source != elastic.ElasticsearchConfigSourceElasticsearch { - return + return false } if v, ok := nodeAlreadyUnknown[clusterID]; ok && v { - return + return false } queueConfig := queue.GetOrInitConfig(elastic.QueueElasticIndexState) if queueConfig.Labels == nil { @@ -846,6 +849,7 @@ func setNodeUnknown(clusterID string) { } nodeAlreadyUnknown[clusterID] = true + return true } func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) error { esConfig := elastic.GetConfig(clusterID) diff --git a/modules/elastic/module.go b/modules/elastic/module.go index 52264b0cc..a1f11fb84 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -693,7 +693,18 @@ func (module *ElasticModule) refreshAllClusterMetadata() { log.Trace("update elasticsearch's metadata:", v, ok) if ok { - module.updateNodeInfo(v, false, v.Config.Discovery.Enabled) + cfg := elastic.GetConfigNoPanic(v.Config.ID) + if cfg == nil { + log.Debugf("elasticsearch metadata [%v] has no active config, removing stale metadata", v.Config.ID) + elastic.RemoveInstance(v.Config.ID) + elastic.RemoveHostsByClusterID(v.Config.ID) + return true + } + v.Config = cfg + if !cfg.Enabled || (cfg.MetadataConfigs != nil && !cfg.MetadataConfigs.MetadataRefresh.Enabled) { + return true + } + module.updateNodeInfo(v, false, cfg.Discovery.Enabled) } return true }) @@ -706,6 +717,17 @@ func (module *ElasticModule) refreshAllClusterAlias(force bool) { } v, ok := value.(*elastic.ElasticsearchMetadata) if ok { + cfg := elastic.GetConfigNoPanic(v.Config.ID) + if cfg == nil { + log.Debugf("elasticsearch metadata [%v] has no active config, removing stale metadata", v.Config.ID) + elastic.RemoveInstance(v.Config.ID) + elastic.RemoveHostsByClusterID(v.Config.ID) + return true + } + v.Config = cfg + if !cfg.Enabled || (cfg.MetadataConfigs != nil && !cfg.MetadataConfigs.MetadataRefresh.Enabled) { + return true + } updateAliases(v, force) } return true diff --git a/modules/elastic/schema.go b/modules/elastic/schema.go index a437af49f..80119c1c1 100755 --- a/modules/elastic/schema.go +++ b/modules/elastic/schema.go @@ -35,6 +35,7 @@ import ( "sync" "unicode" + "infini.sh/framework/core/elastic" "infini.sh/framework/core/global" "github.com/buger/jsonparser" @@ -124,6 +125,62 @@ func parseAnnotation(mapping []util.Annotation) string { return json } +func ensureDefaultStringDynamicTemplates(mappingData map[string]interface{}) { + if mappingData == nil { + return + } + if _, ok := mappingData["dynamic_templates"]; ok { + return + } + mappingData["dynamic_templates"] = []interface{}{ + util.MapStr{ + "strings": util.MapStr{ + "match_mapping_type": "string", + "mapping": util.MapStr{ + "type": "keyword", + "ignore_above": 256, + }, + }, + }, + } +} + +func containsKeyDeep(value interface{}, targetKey string) bool { + switch v := value.(type) { + case map[string]interface{}: + for key, nested := range v { + if key == targetKey { + return true + } + if containsKeyDeep(nested, targetKey) { + return true + } + } + case []interface{}: + for _, nested := range v { + if containsKeyDeep(nested, targetKey) { + return true + } + } + } + return false +} + +func shouldRefreshExistingTemplate(client elastic.API, templateName string, mappingData map[string]interface{}) bool { + if mappingData == nil { + return false + } + if _, ok := mappingData["dynamic_templates"]; !ok { + return false + } + template, err := client.GetTemplate(templateName) + if err != nil { + log.Warnf("failed to inspect existing template [%s]: %v", templateName, err) + return false + } + return !containsKeyDeep(template, "dynamic_templates") +} + func initIndexName(t interface{}, indexName string) string { pkg, ojbType := util.GetTypeAndPackageName(t, true) key := fmt.Sprintf("%s-%s", pkg, ojbType) @@ -181,6 +238,7 @@ func (handler *ElasticORM) RegisterSchemaWithName(t interface{}, indexName strin } return err } + ensureDefaultStringDynamicTemplates(mappingData) template, err := handler.Client.BuildTemplate(indexName+"*", nil, mappingData) if err != nil { if handler.Config.PanicOnInitSchemaError { @@ -228,6 +286,44 @@ func (handler *ElasticORM) RegisterSchemaWithName(t interface{}, indexName strin //init index _ = handler.tryCreateInitIndex(t, indexName) + } else if handler.Config.BuildTemplateForObject { + jsonFormat := `{ %s }` + mapping := getIndexMapping(t) + js := parseAnnotation(mapping) + json := fmt.Sprintf(jsonFormat, quoteJson(js)) + + var mappingData map[string]interface{} + err = util.FromJSONBytes([]byte(json), &mappingData) + if err != nil { + if handler.Config.PanicOnInitSchemaError { + panic(err) + } + return err + } + ensureDefaultStringDynamicTemplates(mappingData) + if shouldRefreshExistingTemplate(handler.Client, indexTemplate, mappingData) { + template, err := handler.Client.BuildTemplate(indexName+"*", nil, mappingData) + if err != nil { + if handler.Config.PanicOnInitSchemaError { + panic(err) + } + return err + } + data, err := handler.Client.PutTemplate(indexTemplate, template) + if err != nil { + if handler.Config.PanicOnInitSchemaError { + panic(err) + } + return err + } + x, _, _, _ := jsonparser.Get(data, "error") + if x != nil { + log.Errorf("error on update template: %v, %v", indexName, string(x)) + if handler.Config.PanicOnInitSchemaError { + panic(string(data)) + } + } + } } return err } diff --git a/modules/elastic/schema_test.go b/modules/elastic/schema_test.go index 518da84a1..ae2c39330 100644 --- a/modules/elastic/schema_test.go +++ b/modules/elastic/schema_test.go @@ -91,3 +91,19 @@ func TestQuoteWithUnderscore(t *testing.T) { json := quoteJson(js) assert.Equal(t, json, `{ "properties":{ "id": { "type": "keyword" },"created": { "type": "date" },"updated": { "type": "date" },"_system": { "type": "object" },"name": { "type": "keyword" } } }`) } + +func TestEnsureDefaultStringDynamicTemplates(t *testing.T) { + mapping := map[string]interface{}{ + "properties": map[string]interface{}{ + "timestamp": map[string]interface{}{ + "type": "date", + }, + }, + } + + ensureDefaultStringDynamicTemplates(mapping) + + templates, ok := mapping["dynamic_templates"].([]interface{}) + assert.Equal(t, ok, true) + assert.Equal(t, len(templates), 1) +} diff --git a/modules/metrics/metrics.go b/modules/metrics/metrics.go index 1e2444b9a..5fae56cc6 100755 --- a/modules/metrics/metrics.go +++ b/modules/metrics/metrics.go @@ -171,7 +171,7 @@ func (module *MetricsModule) CollectAgentMetric() { Type: "interval", Interval: "10s", Task: func(ctx context.Context) { - log.Debug("collecting instance metrics") + log.Trace("collecting instance metrics") agentM.Collect() }, } @@ -202,7 +202,7 @@ func (module *MetricsModule) CollectHostMetric() { Type: "interval", Interval: "10s", Task: func(ctx context.Context) { - log.Debug("collecting network metrics") + log.Trace("collecting network metrics") netM.Collect() }, } diff --git a/modules/queue/disk_queue/cleanup.go b/modules/queue/disk_queue/cleanup.go index 1fed12942..3456d881f 100644 --- a/modules/queue/disk_queue/cleanup.go +++ b/modules/queue/disk_queue/cleanup.go @@ -92,7 +92,7 @@ func (module *DiskQueue) deleteUnusedFiles(queueID string, fileNum int64) { fileStartToDelete := fileNum - module.cfg.Retention.MaxNumOfLocalFiles if fileStartToDelete <= 0 || consumers <= 0 || eSegmentNum < 0 { - log.Debugf("queue: %v, no consumers or consumer/s3 already ahead of this file, %v, %v, %v", queueID, fileStartToDelete, consumers, eSegmentNum) + log.Tracef("queue: %v, no consumers or consumer/s3 already ahead of this file, %v, %v, %v", queueID, fileStartToDelete, consumers, eSegmentNum) return } diff --git a/modules/queue/disk_queue/compress.go b/modules/queue/disk_queue/compress.go index b03fdccfb..63404a21e 100644 --- a/modules/queue/disk_queue/compress.go +++ b/modules/queue/disk_queue/compress.go @@ -104,7 +104,7 @@ func (module *DiskQueue) compressFiles(queueID string, fileNum int64) { //skip compress file if fileStartToCompress <= 0 || (module.cfg.SkipZeroConsumers && consumers <= 0) || fileStartToCompress <= lastCompressedFileNum { - log.Debugf("skip compress %v", queueID) + log.Tracef("skip compress %v", queueID) return } diff --git a/modules/queue/disk_queue/consumer.go b/modules/queue/disk_queue/consumer.go index c18bbd83e..5811fc1aa 100644 --- a/modules/queue/disk_queue/consumer.go +++ b/modules/queue/disk_queue/consumer.go @@ -236,7 +236,7 @@ READ_MSG: } return messages, false, err } - log.Debugf("queue:%v, offset:%v,%v, msgSize:%v", d.queue, d.segment, d.readPos, msgSize) + log.Tracef("queue:%v, offset:%v,%v, msgSize:%v", d.queue, d.segment, d.readPos, msgSize) if int32(msgSize) < d.mCfg.MinMsgSize || int32(msgSize) > d.mCfg.MaxMsgSize { //current have changes, reload file with new position newFileSize := d.getFileSize() @@ -332,9 +332,9 @@ READ_MSG: //still working on the same file if d.diskQueue.writeSegmentNum == d.segment { time.Sleep(100 * time.Millisecond) // Prevent catching up too quickly. - log.Debugf("invalid message size detected. this might be due to a dirty read as the file was being written while open. reloading segment: %d", d.segment) + log.Tracef("invalid message size detected. this might be due to a dirty read as the file was being written while open. reloading segment: %d", d.segment) } else { - log.Debugf("invalid message size detected. this might be due to a partial file load. reloading segment: %d", d.segment) + log.Tracef("invalid message size detected. this might be due to a partial file load. reloading segment: %d", d.segment) } d.readPos = previousPos diff --git a/modules/queue/disk_queue/diskqueue.go b/modules/queue/disk_queue/diskqueue.go index d29a8f1da..e3f8c9278 100644 --- a/modules/queue/disk_queue/diskqueue.go +++ b/modules/queue/disk_queue/diskqueue.go @@ -69,6 +69,8 @@ import ( "infini.sh/framework/core/util/zstd" ) +const bytesPerMiB = 1024 * 1024 + // providing a filesystem backed FIFO queue type DiskBasedQueue struct { sync.RWMutex @@ -118,6 +120,8 @@ type DiskBasedQueue struct { // NewDiskQueue instantiates a new instance of DiskBasedQueue, retrieving metadata // from the filesystem and starting the read ahead goroutine func NewDiskQueueByConfig(name, dataPath string, cfg *DiskQueueConfig) *DiskBasedQueue { + normalizeDiskQueueConfig(cfg) + d := DiskBasedQueue{ name: name, dataPath: dataPath, @@ -177,7 +181,8 @@ func (d *DiskBasedQueue) ReadChan() <-chan []byte { // Put writes a []byte to the queue func (d *DiskBasedQueue) Put(data []byte) WriteResponse { - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(d.cfg.WriteTimeoutInMS)*time.Millisecond) + writeTimeout := d.getWriteTimeout(len(data)) + ctx, cancel := context.WithTimeout(context.Background(), writeTimeout) defer cancel() size := int64(len(data)) @@ -232,7 +237,7 @@ func (d *DiskBasedQueue) Put(data []byte) WriteResponse { switch res.Error { case context.DeadlineExceeded: // Handle timeout error specifically - res.Error = fmt.Errorf("operation timed out: %w", res.Error) + res.Error = fmt.Errorf("operation timed out after %s waiting for disk queue writer availability: %w", writeTimeout, res.Error) case context.Canceled: // Handle cancellation error specifically res.Error = fmt.Errorf("operation was canceled: %w", res.Error) @@ -244,6 +249,28 @@ func (d *DiskBasedQueue) Put(data []byte) WriteResponse { } } +func (d *DiskBasedQueue) getWriteTimeout(payloadSize int) time.Duration { + timeoutInMS := defaultWriteTimeoutInMS + if d != nil && d.cfg != nil && d.cfg.WriteTimeoutInMS > 0 { + timeoutInMS = d.cfg.WriteTimeoutInMS + } + + if payloadSize > 0 { + payloadMiB := int64((payloadSize + bytesPerMiB - 1) / bytesPerMiB) + timeoutInMS += payloadMiB * adaptiveWriteTimeoutPerPayloadMiBInMS + } + + if d != nil && len(d.writeChan) > 0 { + timeoutInMS += int64(len(d.writeChan)) * adaptiveWriteTimeoutPerQueuedWriteInMS + } + + if timeoutInMS > maxAdaptiveWriteTimeoutInMS { + timeoutInMS = maxAdaptiveWriteTimeoutInMS + } + + return time.Duration(timeoutInMS) * time.Millisecond +} + // Close cleans up the queue and persists metadata func (d *DiskBasedQueue) Close() error { err := d.exit(false) diff --git a/modules/queue/disk_queue/diskqueue_test.go b/modules/queue/disk_queue/diskqueue_test.go new file mode 100644 index 000000000..a0c0e3bb6 --- /dev/null +++ b/modules/queue/disk_queue/diskqueue_test.go @@ -0,0 +1,40 @@ +package queue + +import ( + "testing" + "time" +) + +func TestGetWriteTimeoutIncludesPayloadAndBacklog(t *testing.T) { + dq := &DiskBasedQueue{ + cfg: &DiskQueueConfig{WriteTimeoutInMS: defaultWriteTimeoutInMS}, + writeChan: make(chan []byte, defaultWriteChanBuffer), + } + + dq.writeChan <- []byte("a") + dq.writeChan <- []byte("b") + + timeout := dq.getWriteTimeout(3 * bytesPerMiB) + + expected := time.Duration(defaultWriteTimeoutInMS+3*adaptiveWriteTimeoutPerPayloadMiBInMS+2*adaptiveWriteTimeoutPerQueuedWriteInMS) * time.Millisecond + if timeout != expected { + t.Fatalf("unexpected write timeout: got %s want %s", timeout, expected) + } +} + +func TestGetWriteTimeoutCapsAtMaximum(t *testing.T) { + dq := &DiskBasedQueue{ + cfg: &DiskQueueConfig{WriteTimeoutInMS: defaultWriteTimeoutInMS}, + writeChan: make(chan []byte, defaultWriteChanBuffer), + } + + for i := 0; i < cap(dq.writeChan); i++ { + dq.writeChan <- []byte("x") + } + + timeout := dq.getWriteTimeout(64 * bytesPerMiB) + expected := time.Duration(maxAdaptiveWriteTimeoutInMS) * time.Millisecond + if timeout != expected { + t.Fatalf("unexpected capped timeout: got %s want %s", timeout, expected) + } +} diff --git a/modules/queue/disk_queue/module.go b/modules/queue/disk_queue/module.go index d7b1d8ee9..deb3fe600 100644 --- a/modules/queue/disk_queue/module.go +++ b/modules/queue/disk_queue/module.go @@ -124,8 +124,33 @@ type CompressConfig struct { Level int `config:"level"` } +const ( + defaultWriteTimeoutInMS int64 = 60 * 1000 + defaultWriteChanBuffer = 16 + minRecommendedWriteTimeoutInMS int64 = 15 * 1000 + maxAdaptiveWriteTimeoutInMS int64 = 5 * 60 * 1000 + adaptiveWriteTimeoutPerQueuedWriteInMS int64 = 3 * 1000 + adaptiveWriteTimeoutPerPayloadMiBInMS int64 = 5 * 1000 +) + var preventRead bool +func normalizeDiskQueueConfig(cfg *DiskQueueConfig) { + if cfg == nil { + return + } + + if cfg.WriteTimeoutInMS <= 0 { + cfg.WriteTimeoutInMS = defaultWriteTimeoutInMS + } else if cfg.WriteTimeoutInMS < minRecommendedWriteTimeoutInMS { + log.Warnf("disk_queue write timeout may be too small on slow disks: %dms", cfg.WriteTimeoutInMS) + } + + if cfg.WriteChanBuffer <= 0 { + cfg.WriteChanBuffer = defaultWriteChanBuffer + } +} + func checkCapacity(cfg *DiskQueueConfig) error { if cfg.CheckDiskCapacityRetryDelayInMs <= 0 { @@ -233,14 +258,14 @@ func (module *DiskQueue) Setup() { MinMsgSize: 1, MaxMsgSize: 104857600, //100MB MaxBytesPerFile: 100 * 1024 * 1024, //100MB - WriteTimeoutInMS: 1000, //1s - CheckDiskCapacityRetryDelayInMs: 10 * 000, //10s + WriteTimeoutInMS: defaultWriteTimeoutInMS, + CheckDiskCapacityRetryDelayInMs: 10 * 000, //10s EOFRetryDelayInMs: 500, SyncEveryRecords: 1000, SyncTimeoutInMS: 1000, NotifyChanBuffer: 100, ReadChanBuffer: 0, - WriteChanBuffer: 0, + WriteChanBuffer: defaultWriteChanBuffer, WarningFreeBytes: 10 * 1024 * 1024 * 1024, ReservedFreeBytes: 5 * 1024 * 1024 * 1024, PrepareFilesToRead: true, @@ -262,6 +287,8 @@ func (module *DiskQueue) Setup() { panic(err) } + normalizeDiskQueueConfig(module.cfg) + if !module.cfg.Enabled { return } diff --git a/modules/queue/disk_queue/module_test.go b/modules/queue/disk_queue/module_test.go new file mode 100644 index 000000000..d17b0875b --- /dev/null +++ b/modules/queue/disk_queue/module_test.go @@ -0,0 +1,32 @@ +package queue + +import "testing" + +func TestNormalizeDiskQueueConfigAppliesRobustWriteDefaults(t *testing.T) { + cfg := &DiskQueueConfig{} + + normalizeDiskQueueConfig(cfg) + + if cfg.WriteTimeoutInMS != defaultWriteTimeoutInMS { + t.Fatalf("unexpected write timeout: %d", cfg.WriteTimeoutInMS) + } + if cfg.WriteChanBuffer != defaultWriteChanBuffer { + t.Fatalf("unexpected write chan buffer: %d", cfg.WriteChanBuffer) + } +} + +func TestNormalizeDiskQueueConfigKeepsExplicitWriteSettings(t *testing.T) { + cfg := &DiskQueueConfig{ + WriteTimeoutInMS: 45 * 1000, + WriteChanBuffer: 64, + } + + normalizeDiskQueueConfig(cfg) + + if cfg.WriteTimeoutInMS != 45*1000 { + t.Fatalf("write timeout should be preserved, got %d", cfg.WriteTimeoutInMS) + } + if cfg.WriteChanBuffer != 64 { + t.Fatalf("write chan buffer should be preserved, got %d", cfg.WriteChanBuffer) + } +} diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index 5c7a32c6d..adfa1cc11 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -322,7 +322,9 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { } } else { cfgs := queue.GetConfigBySelector(&processor.config.Selector) - log.Debugf("filter queue by:%v, num of queues:%v", processor.config.Selector.ToString(), len(cfgs)) + if global.Env().IsDebug { + log.Tracef("filter queue by:%v, num of queues:%v", processor.config.Selector.ToString(), len(cfgs)) + } for _, v := range cfgs { if global.Env().IsDebug { log.Tracef("checking queue: %v", v) @@ -484,7 +486,7 @@ func (processor *BulkIndexingProcessor) NewBulkWorker(parentContext *pipeline.Co continue } - log.Debugf("starting worker:[%v], queue:[%v], slice_id:%v, host:[%v]", workerID, qConfig.Name, sliceID, preferedHost) + log.Tracef("starting worker:[%v], queue:[%v], slice_id:%v, host:[%v]", workerID, qConfig.Name, sliceID, preferedHost) ctx1 := &pipeline.Context{} ctx1.Set("key", key) @@ -758,7 +760,9 @@ func (processor *BulkIndexingProcessor) NewSlicedBulkWorker(ctx *pipeline.Contex log.Errorf("should not submit this bulk request, worker[%v], queue:[%v], slice:[%v], offset:[%v]->[%v],%v, msg:%v", workerID, qConfig.ID, sliceID, committedOffset, offset, err, mainBuf.GetMessageCount()) } } - log.Debugf("exit worker[%v], message count[%d], queue:[%v], slice_id:%v", workerID, mainBuf.GetMessageCount(), qConfig.ID, sliceID) + if global.Env().IsDebug { + log.Tracef("exit worker[%v], message count[%d], queue:[%v], slice_id:%v", workerID, mainBuf.GetMessageCount(), qConfig.ID, sliceID) + } }() if global.Env().IsDebug { @@ -875,7 +879,13 @@ READ_DOCS: consumerConfig.KeepActive() messages, timeout, err := consumerInstance.FetchMessages(ctx1, consumerConfig.FetchMaxMessages) stats.IncrementBy("queue", qConfig.ID+".msg_fetched_from_queue", int64(len(messages))) - log.Debugf("slice worker, worker:[%v], [%v][%v][%v][%v] fetched message:%v,ctx:%v,timeout:%v,err:%v", workerID, qConfig.Name, consumerConfig.Group, consumerConfig.Name, sliceID, len(messages), ctx1.String(), timeout, err) + if err != nil || len(messages) > 0 { + if qConfig.Name == "bulk_requests" { + log.Tracef("slice worker, worker:[%v], [%v][%v][%v][%v] fetched message:%v,ctx:%v,timeout:%v,err:%v", workerID, qConfig.Name, consumerConfig.Group, consumerConfig.Name, sliceID, len(messages), ctx1.String(), timeout, err) + } else { + log.Debugf("slice worker, worker:[%v], [%v][%v][%v][%v] fetched message:%v,ctx:%v,timeout:%v,err:%v", workerID, qConfig.Name, consumerConfig.Group, consumerConfig.Name, sliceID, len(messages), ctx1.String(), timeout, err) + } + } if err != nil { if strings.Contains(err.Error(), "dirty_read") || err.Error() == "EOF" || err.Error() == "unexpected EOF" { ctx.CancelTask() @@ -1026,7 +1036,7 @@ READ_DOCS: if offset != nil && committedOffset != nil && !offset.Equals(*committedOffset) { err := consumerInstance.CommitOffset(*offset) if err != nil { - log.Errorf("🔧 offset commit failed, worker:[%v], queue:[%v], slice:[%v], offset:[%v], err:%v", workerID, qConfig.Name, sliceID, *offset, err) + log.Errorf("offset commit failed, worker:[%v], queue:[%v], slice:[%v], offset:[%v], err:%v", workerID, qConfig.Name, sliceID, *offset, err) panic(err) } @@ -1035,11 +1045,11 @@ READ_DOCS: } // fix: update committedOffset immediately after successful commit, to ensure state consistency committedOffset = offset - log.Debugf("🔧 offset committed successfully, worker:[%v], queue:[%v], slice:[%v], offset:[%v]", workerID, qConfig.Name, sliceID, *offset) - } else { if global.Env().IsDebug { - log.Debugf("🔧 offset not changed, skip commit, worker:[%v], queue:[%v], slice:[%v], offset:[%v], committed:[%v]", workerID, qConfig.Name, sliceID, offset, committedOffset) + log.Tracef("offset committed, worker:[%v], queue:[%v], slice:[%v], offset:[%v]", workerID, qConfig.Name, sliceID, *offset) } + } else { + // skip unchanged offset silently to avoid noisy debug logs } // fix: this code is moved to loop outside (line 970) to avoid updating offset in the middle of bulk submission // offset = &pop.NextOffset @@ -1079,7 +1089,7 @@ CLEAN_BUFFER: } if global.Env().IsDebug { - log.Debugf("cleanup buffer, queue:[%v], slice_id:%v, offset [%v]-[%v], bulk failed (host: %v, err: %v)", qConfig.ID, sliceID, committedOffset, offset, host, err) + log.Tracef("cleanup buffer, queue:[%v], slice_id:%v, offset [%v]-[%v], bulk failed (host: %v, err: %v)", qConfig.ID, sliceID, committedOffset, offset, host, err) } lastCommit = time.Now() // check bulk result, if ok, then commit offset, or retry non-200 requests, or save failure offset From 8231706b341f3b9b3a1a5994e913d1eb93f54732 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 16 May 2026 08:57:04 +0800 Subject: [PATCH 007/137] improve: add hash and terms partition --- core/elastic/index.go | 63 ++++ core/elastic/index_test.go | 17 + core/elastic/partition.go | 649 ++++++++++++++++++++++++++++++++++--- 3 files changed, 689 insertions(+), 40 deletions(-) diff --git a/core/elastic/index.go b/core/elastic/index.go index 6816e9173..27768169e 100755 --- a/core/elastic/index.go +++ b/core/elastic/index.go @@ -29,6 +29,8 @@ import ( "github.com/buger/jsonparser" "github.com/segmentio/encoding/json" "infini.sh/framework/core/util" + "sort" + "strconv" "strings" "time" ) @@ -218,6 +220,67 @@ type AggregationResponse struct { Value interface{} `json:"value,omitempty"` } +func (a *AggregationResponse) UnmarshalJSON(data []byte) error { + type alias struct { + Buckets json.RawMessage `json:"buckets,omitempty"` + Value interface{} `json:"value,omitempty"` + } + + var aux alias + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + a.Value = aux.Value + + buckets := bytes.TrimSpace(aux.Buckets) + if len(buckets) == 0 || bytes.Equal(buckets, []byte("null")) { + a.Buckets = nil + return nil + } + + switch buckets[0] { + case '[': + return json.Unmarshal(buckets, &a.Buckets) + case '{': + keyedBuckets := map[string]BucketBase{} + if err := json.Unmarshal(buckets, &keyedBuckets); err != nil { + return err + } + + keys := make([]string, 0, len(keyedBuckets)) + for key := range keyedBuckets { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + return compareBucketKeys(keys[i], keys[j]) + }) + + a.Buckets = make([]BucketBase, 0, len(keys)) + for _, key := range keys { + bucket := keyedBuckets[key] + if bucket == nil { + bucket = BucketBase{} + } + if _, ok := bucket["key"]; !ok { + bucket["key"] = key + } + a.Buckets = append(a.Buckets, bucket) + } + return nil + default: + return nil + } +} + +func compareBucketKeys(left, right string) bool { + leftInt, leftErr := strconv.ParseInt(left, 10, 64) + rightInt, rightErr := strconv.ParseInt(right, 10, 64) + if leftErr == nil && rightErr == nil { + return leftInt < rightInt + } + return left < right +} + type ResponseBase struct { RawResult *util.Result `json:"-"` StatusCode int `json:"-"` diff --git a/core/elastic/index_test.go b/core/elastic/index_test.go index af98fdc3a..399a9d6bc 100644 --- a/core/elastic/index_test.go +++ b/core/elastic/index_test.go @@ -29,6 +29,23 @@ import ( "github.com/segmentio/encoding/json" ) +func TestAggregationResponseUnmarshalKeyedBuckets(t *testing.T) { + var agg AggregationResponse + err := json.Unmarshal([]byte(`{"buckets":{"0":{"doc_count":1740269},"1":{"doc_count":42}}}`), &agg) + if err != nil { + t.Fatalf("unexpected unmarshal error: %v", err) + } + if len(agg.Buckets) != 2 { + t.Fatalf("unexpected bucket count: %d", len(agg.Buckets)) + } + if agg.Buckets[0]["key"] != "0" || agg.Buckets[0]["doc_count"] != float64(1740269) { + t.Fatalf("unexpected first bucket: %#v", agg.Buckets[0]) + } + if agg.Buckets[1]["key"] != "1" || agg.Buckets[1]["doc_count"] != float64(42) { + t.Fatalf("unexpected second bucket: %#v", agg.Buckets[1]) + } +} + func TestIndexDocument_GetStringFieldFromSource(t *testing.T) { tests := []struct { name string diff --git a/core/elastic/partition.go b/core/elastic/partition.go index 63d474fb1..d4c0127d6 100644 --- a/core/elastic/partition.go +++ b/core/elastic/partition.go @@ -32,6 +32,7 @@ import ( "fmt" "math" "net/http" + "sort" "strconv" "strings" @@ -40,12 +41,14 @@ import ( ) type PartitionQuery struct { - IndexName string `json:"index_name"` - FieldType string `json:"field_type"` - FieldName string `json:"field_name"` - Step interface{} `json:"step"` - Filter interface{} `json:"filter"` - DocType string `json:"doc_type"` + IndexName string `json:"index_name"` + FieldType string `json:"field_type"` + FieldName string `json:"field_name"` + Strategy string `json:"strategy,omitempty"` + Step interface{} `json:"step,omitempty"` + PartitionCount int `json:"partition_count,omitempty"` + Filter interface{} `json:"filter"` + DocType string `json:"doc_type"` } type PartitionInfo struct { @@ -54,6 +57,8 @@ type PartitionInfo struct { End float64 `json:"end"` Filter map[string]interface{} `json:"filter"` Docs int64 `json:"docs"` + Label string `json:"label,omitempty"` + Values []string `json:"values,omitempty"` Other bool } @@ -68,6 +73,11 @@ const ( PartitionByDate = "date" PartitionByKeyword = "keyword" PartitionByNumber = "number" + + PartitionStrategyStep = "step" + PartitionStrategyQuantile = "quantile" + PartitionStrategyTerms = "terms" + PartitionStrategyHash = "hash" ) func GetPartitions(q *PartitionQuery, client API) ([]PartitionInfo, error) { @@ -100,32 +110,6 @@ func GetPartitions(q *PartitionQuery, client API) ([]PartitionInfo, error) { switch q.FieldType { case PartitionByDate, PartitionByNumber: - var step float64 - if q.FieldType == PartitionByDate { - if stepV, ok := q.Step.(string); !ok { - return nil, fmt.Errorf("expect step value of string type since filedtype is %s", PartitionByDate) - } else { - du, err := util.ParseDuration(stepV) - if err != nil { - return nil, fmt.Errorf("parse step duration error: %w", err) - } - step = float64(du.Milliseconds()) - } - } else { - switch q.Step.(type) { - case float64: - step = q.Step.(float64) - case string: - v, err := strconv.Atoi(q.Step.(string)) - if err != nil { - return nil, fmt.Errorf("convert step error: %w", err) - } - step = float64(v) - default: - return nil, fmt.Errorf("invalid parameter step: %v", q.Step) - } - } - result, err := getBoundValues(client, q.IndexName, q.FieldName, vFilter) if err != nil { return nil, err @@ -138,23 +122,110 @@ func GetPartitions(q *PartitionQuery, client API) ([]PartitionInfo, error) { var ( partitions []PartitionInfo ) - partitions, err = getPartitionsByAgg(client, q.IndexName, q.FieldName, q.FieldType, step, vFilter) - if err != nil { - return nil, err + + switch normalizePartitionStrategy(q.Strategy) { + case PartitionStrategyStep: + step, err := parsePartitionStep(q.FieldType, q.Step) + if err != nil { + return nil, err + } + partitions, err = getPartitionsByAgg(client, q.IndexName, q.FieldName, q.FieldType, step, vFilter) + if err != nil { + return nil, err + } + case PartitionStrategyQuantile: + partitions, err = getPartitionsByQuantile(client, q.IndexName, q.FieldName, q.FieldType, q.PartitionCount, result.Min, result.Max, vFilter) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unsupported partition strategy: %s", q.Strategy) } + if result.Null > 0 { partitions = append(partitions, PartitionInfo{ Filter: result.NotExistsFilter, Other: true, + Label: "Missing values", Docs: result.Null, }) } return partitions, nil + case PartitionByKeyword: + var ( + partitions []PartitionInfo + err error + ) + switch normalizePartitionStrategy(q.Strategy) { + case PartitionStrategyTerms: + partitions, err = getPartitionsByTerms(client, q.IndexName, q.FieldName, q.PartitionCount, vFilter) + if err != nil { + return nil, err + } + case PartitionStrategyHash: + partitions, err = getPartitionsByHash(client, q.IndexName, q.FieldName, q.PartitionCount, vFilter) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unsupported partition strategy: %s", q.Strategy) + } + + missingPartition, err := getMissingPartition(client, q.IndexName, q.FieldName, vFilter) + if err != nil { + return nil, err + } + if missingPartition != nil { + partitions = append(partitions, *missingPartition) + } + return partitions, nil default: return nil, fmt.Errorf("unsupported field type: %s", q.FieldType) } } +func normalizePartitionStrategy(strategy string) string { + switch strings.ToLower(strings.TrimSpace(strategy)) { + case "", PartitionStrategyStep: + return PartitionStrategyStep + case PartitionStrategyQuantile: + return PartitionStrategyQuantile + case PartitionStrategyTerms: + return PartitionStrategyTerms + case PartitionStrategyHash: + return PartitionStrategyHash + default: + return strings.ToLower(strings.TrimSpace(strategy)) + } +} + +func parsePartitionStep(fieldType string, stepValue interface{}) (float64, error) { + if fieldType == PartitionByDate { + stepV, ok := stepValue.(string) + if !ok { + return 0, fmt.Errorf("expect step value of string type since filedtype is %s", PartitionByDate) + } + du, err := util.ParseDuration(stepV) + if err != nil { + return 0, fmt.Errorf("parse step duration error: %w", err) + } + return float64(du.Milliseconds()), nil + } + + switch stepValue.(type) { + case float64: + return stepValue.(float64), nil + case string: + v, err := strconv.Atoi(stepValue.(string)) + if err != nil { + return 0, fmt.Errorf("convert step error: %w", err) + } + return float64(v), nil + default: + return 0, fmt.Errorf("invalid parameter step: %v", stepValue) + } +} + func getPartitionsByAgg(client API, indexName string, fieldName, fieldType string, step float64, filter interface{}) ([]PartitionInfo, error) { queryDsl := util.MapStr{ "size": 0, @@ -182,7 +253,7 @@ func getPartitionsByAgg(client API, indexName string, fieldName, fieldType strin if filter != nil { queryDsl["query"] = filter } - res, err := client.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(queryDsl)) + res, err := searchPartitionWithRawQueryDSL(client, indexName, queryDsl) if err != nil { return nil, err } @@ -217,13 +288,321 @@ func getPartitionsByAgg(client API, indexName string, fieldName, fieldType strin Docs: int64(docCount), Other: false, } - partition.Filter = buildPartitionFilter(min, max, fieldName, fieldType, filter) + partition.Filter = buildBoundedPartitionFilter(min, max, fieldName, fieldType, filter) partitions = append(partitions, partition) } } return partitions, nil } +func getPartitionsByQuantile(client API, indexName string, fieldName, fieldType string, partitionCount int, min, max float64, filter interface{}) ([]PartitionInfo, error) { + if partitionCount <= 0 { + return nil, fmt.Errorf("invalid parameter partition_count: %d", partitionCount) + } + + boundaries, err := getQuantileBoundaries(client, indexName, fieldName, partitionCount, min, max, filter) + if err != nil { + return nil, err + } + partitions := buildQuantilePartitions(boundaries, fieldName, fieldType, filter) + if len(partitions) == 0 { + return nil, nil + } + + counts, err := getPartitionDocCounts(client, indexName, partitions) + if err != nil { + return nil, err + } + + filtered := make([]PartitionInfo, 0, len(partitions)) + for i := range partitions { + partitions[i].Docs = counts[i] + if partitions[i].Docs <= 0 { + continue + } + filtered = append(filtered, partitions[i]) + } + return filtered, nil +} + +func getPartitionsByTerms(client API, indexName, fieldName string, partitionCount int, filter interface{}) ([]PartitionInfo, error) { + if partitionCount <= 0 { + return nil, fmt.Errorf("invalid parameter partition_count: %d", partitionCount) + } + + queryDsl := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "partitions": util.MapStr{ + "terms": util.MapStr{ + "field": fieldName, + "size": partitionCount, + }, + }, + }, + } + if filter != nil { + queryDsl["query"] = filter + } + + res, err := searchPartitionWithRawQueryDSL(client, indexName, queryDsl) + if err != nil { + return nil, err + } + + var ( + partitions []PartitionInfo + values []string + ) + if partitionsAgg, ok := res.Aggregations["partitions"]; ok { + for idx, bucket := range partitionsAgg.Buckets { + value := fmt.Sprintf("%v", bucket["key"]) + docCount := util.GetInt64Value(bucket["doc_count"]) + if docCount <= 0 { + continue + } + values = append(values, value) + partitions = append(partitions, PartitionInfo{ + Key: float64(idx), + Docs: docCount, + Label: value, + Values: []string{value}, + Filter: buildExactTermPartitionFilter(value, fieldName, filter), + }) + } + } + + sumOtherDocCount, _ := jsonparser.GetInt(res.RawResult.Body, "aggregations", "partitions", "sum_other_doc_count") + if sumOtherDocCount > 0 { + partitions = append(partitions, PartitionInfo{ + Key: float64(len(partitions)), + Docs: sumOtherDocCount, + Label: "Other terms", + Values: append([]string(nil), values...), + Filter: buildOtherTermsPartitionFilter(values, fieldName, filter), + Other: true, + }) + } + + return partitions, nil +} + +func getPartitionsByHash(client API, indexName, fieldName string, partitionCount int, filter interface{}) ([]PartitionInfo, error) { + if partitionCount <= 0 { + return nil, fmt.Errorf("invalid parameter partition_count: %d", partitionCount) + } + + partitions := make([]PartitionInfo, 0, partitionCount) + for idx := 0; idx < partitionCount; idx++ { + partitions = append(partitions, PartitionInfo{ + Key: float64(idx), + Label: fmt.Sprintf("Hash %d/%d", idx+1, partitionCount), + Filter: buildHashPartitionFilter(idx, partitionCount, fieldName, filter), + }) + } + + counts, err := getPartitionDocCounts(client, indexName, partitions) + if err != nil { + return nil, err + } + + filtered := make([]PartitionInfo, 0, len(partitions)) + for idx := range partitions { + partitions[idx].Docs = counts[idx] + if partitions[idx].Docs <= 0 { + continue + } + filtered = append(filtered, partitions[idx]) + } + return filtered, nil +} + +func getQuantileBoundaries(client API, indexName, fieldName string, partitionCount int, min, max float64, filter interface{}) ([]float64, error) { + percents := buildQuantilePercents(partitionCount) + if len(percents) == 0 { + return []float64{min, max}, nil + } + + queryDsl := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "partition_percentiles": util.MapStr{ + "percentiles": util.MapStr{ + "field": fieldName, + "percents": percents, + "keyed": false, + }, + }, + }, + } + if filter != nil { + queryDsl["query"] = filter + } + + res, err := searchPartitionWithRawQueryDSL(client, indexName, queryDsl) + if err != nil { + return nil, err + } + + boundaries := make([]float64, 0, len(percents)+2) + boundaries = append(boundaries, min) + _, err = jsonparser.ArrayEach(res.RawResult.Body, func(value []byte, _ jsonparser.ValueType, _ int, err error) { + if err != nil { + return + } + boundary, parseErr := jsonparser.GetFloat(value, "value") + if parseErr != nil || math.IsNaN(boundary) || math.IsInf(boundary, 0) { + return + } + boundaries = append(boundaries, boundary) + }, "aggregations", "partition_percentiles", "values") + if err != nil { + return nil, err + } + boundaries = append(boundaries, max) + boundaries = dedupeSortedBoundaries(boundaries) + if len(boundaries) == 1 { + return []float64{boundaries[0], boundaries[0]}, nil + } + return boundaries, nil +} + +func buildQuantilePercents(partitionCount int) []float64 { + if partitionCount <= 1 { + return nil + } + percents := make([]float64, 0, partitionCount-1) + for i := 1; i < partitionCount; i++ { + percents = append(percents, float64(i)*100/float64(partitionCount)) + } + return percents +} + +func dedupeSortedBoundaries(boundaries []float64) []float64 { + if len(boundaries) == 0 { + return nil + } + sort.Float64s(boundaries) + result := make([]float64, 0, len(boundaries)) + for _, boundary := range boundaries { + if len(result) == 0 || !sameBoundary(result[len(result)-1], boundary) { + result = append(result, boundary) + } + } + return result +} + +func sameBoundary(left, right float64) bool { + return math.Abs(left-right) <= 1e-9 +} + +func buildQuantilePartitions(boundaries []float64, fieldName, fieldType string, filter interface{}) []PartitionInfo { + if len(boundaries) < 2 { + return nil + } + + partitions := make([]PartitionInfo, 0, len(boundaries)-1) + if len(boundaries) == 2 { + partitions = append(partitions, PartitionInfo{ + Key: boundaries[1], + Start: boundaries[0], + End: boundaries[1], + Filter: buildOpenPartitionFilter(nil, nil, fieldName, fieldType, filter), + }) + return partitions + } + + for i := 1; i < len(boundaries); i++ { + lower, upper := boundaries[i-1], boundaries[i] + if sameBoundary(lower, upper) { + continue + } + + var lowerRef, upperRef *float64 + if i > 1 { + lowerRef = &lower + } + if i < len(boundaries)-1 { + upperRef = &upper + } + + partitions = append(partitions, PartitionInfo{ + Key: upper, + Start: lower, + End: upper, + Filter: buildOpenPartitionFilter(lowerRef, upperRef, fieldName, fieldType, filter), + }) + } + return partitions +} + +func getPartitionDocCounts(client API, indexName string, partitions []PartitionInfo) ([]int64, error) { + queryDsl := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "partitions": util.MapStr{ + "filters": util.MapStr{ + "filters": buildPartitionFiltersMap(partitions), + }, + }, + }, + } + + res, err := searchPartitionWithRawQueryDSL(client, indexName, queryDsl) + if err != nil { + return nil, err + } + + counts := make([]int64, 0, len(partitions)) + for i := range partitions { + docCount, parseErr := jsonparser.GetInt(res.RawResult.Body, "aggregations", "partitions", "buckets", strconv.Itoa(i), "doc_count") + if parseErr != nil { + return nil, parseErr + } + counts = append(counts, docCount) + } + return counts, nil +} + +func buildPartitionFiltersMap(partitions []PartitionInfo) util.MapStr { + filters := util.MapStr{} + for i, partition := range partitions { + filters[strconv.Itoa(i)] = partition.Filter + } + return filters +} + +func getMissingPartition(client API, indexName, fieldName string, filter interface{}) (*PartitionInfo, error) { + queryDsl := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "missing_field": util.MapStr{ + "filter": buildMissingFieldCondition(fieldName), + }, + }, + } + if filter != nil { + queryDsl["query"] = filter + } + + res, err := searchPartitionWithRawQueryDSL(client, indexName, queryDsl) + if err != nil { + return nil, err + } + + docCount, err := jsonparser.GetInt(res.RawResult.Body, "aggregations", "missing_field", "doc_count") + if err != nil || docCount <= 0 { + return nil, err + } + + return &PartitionInfo{ + Docs: docCount, + Label: "Missing values", + Filter: buildMissingFieldFilter(fieldName, filter), + Other: true, + }, nil +} + // NOTE: we assume GetPartitions returned sorted buckets from ES, if not, we need to manually sort source & target partitions by keys // sourcePartitions & targetPartitions must've been generated with same bucket step & offset func MergePartitions(sourcePartitions []PartitionInfo, targetPartitions []PartitionInfo, fieldName, fieldType string, filter interface{}) []PartitionInfo { @@ -253,7 +632,7 @@ func MergePartitions(sourcePartitions []PartitionInfo, targetPartitions []Partit Docs: util.MaxInt64(source.Docs, target.Docs), Other: false, } - partition.Filter = buildPartitionFilter(partition.Start, partition.End, fieldName, fieldType, filter) + partition.Filter = buildBoundedPartitionFilter(partition.Start, partition.End, fieldName, fieldType, filter) ret = append(ret, partition) sourceIdx += 1 targetIdx += 1 @@ -267,7 +646,7 @@ func MergePartitions(sourcePartitions []PartitionInfo, targetPartitions []Partit return ret } -func buildPartitionFilter(min, max float64, fieldName, fieldType string, filter interface{}) util.MapStr { +func buildBoundedPartitionFilter(min, max float64, fieldName, fieldType string, filter interface{}) util.MapStr { rv := util.MapStr{ "gte": min, "lte": max, @@ -290,9 +669,199 @@ func buildPartitionFilter(min, max float64, fieldName, fieldType string, filter "must": must, }, } +} + +func buildOpenPartitionFilter(lower, upper *float64, fieldName, fieldType string, filter interface{}) util.MapStr { + rv := util.MapStr{} + if lower != nil { + rv["gt"] = *lower + } + if upper != nil { + rv["lte"] = *upper + } + if fieldType == PartitionByDate { + rv["format"] = "epoch_millis" + } + var condition interface{} + if len(rv) == 0 || (len(rv) == 1 && rv["format"] != nil) { + condition = util.MapStr{ + "exists": util.MapStr{ + "field": fieldName, + }, + } + } else { + condition = util.MapStr{ + "range": util.MapStr{ + fieldName: rv, + }, + } + } + must := []interface{}{condition} + if filter != nil { + must = append(must, filter) + } + return util.MapStr{ + "bool": util.MapStr{ + "must": must, + }, + } } +func buildExactTermPartitionFilter(value, fieldName string, filter interface{}) util.MapStr { + return buildMustPartitionFilter([]interface{}{ + util.MapStr{ + "term": util.MapStr{ + fieldName: util.MapStr{ + "value": value, + }, + }, + }, + }, filter) +} + +func buildOtherTermsPartitionFilter(values []string, fieldName string, filter interface{}) util.MapStr { + boolFilter := util.MapStr{ + "must": []interface{}{ + util.MapStr{ + "exists": util.MapStr{ + "field": fieldName, + }, + }, + }, + } + if filter != nil { + boolFilter["must"] = append(boolFilter["must"].([]interface{}), filter) + } + if len(values) > 0 { + boolFilter["must_not"] = []interface{}{ + util.MapStr{ + "terms": util.MapStr{ + fieldName: values, + }, + }, + } + } + return util.MapStr{ + "bool": boolFilter, + } +} + +func buildHashPartitionFilter(partitionID, partitionCount int, fieldName string, filter interface{}) util.MapStr { + fieldLiteral := buildPainlessStringLiteral(fieldName) + return buildMustPartitionFilter([]interface{}{ + util.MapStr{ + "script": util.MapStr{ + "script": util.MapStr{ + "lang": "painless", + "source": fmt.Sprintf("doc[%s].size()!=0 && (((doc[%s].value.hashCode() %% params.partition_count) + params.partition_count) %% params.partition_count) == params.partition_id", fieldLiteral, fieldLiteral), + "params": util.MapStr{ + "partition_count": partitionCount, + "partition_id": partitionID, + }, + }, + }, + }, + }, filter) +} + +func buildPainlessStringLiteral(value string) string { + replacer := strings.NewReplacer(`\`, `\\`, `'`, `\'`) + return "'" + replacer.Replace(value) + "'" +} + +func searchPartitionWithRawQueryDSL(client API, indexName string, queryDsl util.MapStr) (*SearchResponse, error) { + res, err := client.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(queryDsl)) + if err != nil { + return nil, err + } + if err := ensurePartitionSearchResponseOK(res); err != nil { + return nil, err + } + return res, nil +} + +func ensurePartitionSearchResponseOK(res *SearchResponse) error { + if res == nil { + return errors.New("empty search response") + } + if res.StatusCode == 0 || res.StatusCode == http.StatusOK { + return nil + } + if res.RawResult != nil && len(res.RawResult.Body) > 0 { + for _, path := range [][]string{ + {"error", "failed_shards", "[0]", "reason", "caused_by", "reason"}, + {"error", "failed_shards", "[0]", "reason", "reason"}, + {"error", "root_cause", "[0]", "reason"}, + {"error", "reason"}, + } { + if msg, ok := getJSONPathString(res.RawResult.Body, path...); ok && msg != "" { + return errors.New(msg) + } + } + } + if msg := res.Error.Message(); msg != "" { + return errors.New(msg) + } + if res.RawResult != nil && len(res.RawResult.Body) > 0 { + return errors.New(string(res.RawResult.Body)) + } + return fmt.Errorf("unexpected search status: %d", res.StatusCode) +} + +func getJSONPathString(data []byte, path ...string) (string, bool) { + v, err := jsonparser.GetString(data, path...) + if err != nil { + return "", false + } + return v, true +} + +func buildMissingFieldCondition(fieldName string) util.MapStr { + return util.MapStr{ + "bool": util.MapStr{ + "must_not": []interface{}{ + util.MapStr{ + "exists": util.MapStr{ + "field": fieldName, + }, + }, + }, + }, + } +} + +func buildMissingFieldFilter(fieldName string, filter interface{}) util.MapStr { + boolFilter := util.MapStr{ + "must": []interface{}{}, + "must_not": []interface{}{ + util.MapStr{ + "exists": util.MapStr{ + "field": fieldName, + }, + }, + }, + } + if filter != nil { + boolFilter["must"] = append(boolFilter["must"].([]interface{}), filter) + } + return util.MapStr{ + "bool": boolFilter, + } +} + +func buildMustPartitionFilter(mustClauses []interface{}, filter interface{}) util.MapStr { + must := append([]interface{}{}, mustClauses...) + if filter != nil { + must = append(must, filter) + } + return util.MapStr{ + "bool": util.MapStr{ + "must": must, + }, + } +} + func getBoundValues(client API, indexName string, fieldName string, filter interface{}) (*BoundValuesResult, error) { nullFilter := util.MapStr{ "bool": util.MapStr{ @@ -326,7 +895,7 @@ func getBoundValues(client API, indexName string, fieldName string, filter inter if filter != nil { queryDsl["query"] = filter } - res, err := client.SearchWithRawQueryDSL(indexName, util.MustToJSONBytes(queryDsl)) + res, err := searchPartitionWithRawQueryDSL(client, indexName, queryDsl) if err != nil { return nil, err } From d5a190d60036be03230b4403d4702af0f479e923 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 16 May 2026 08:57:44 +0800 Subject: [PATCH 008/137] improve: add test for patition --- core/elastic/partition_test.go | 168 +++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 core/elastic/partition_test.go diff --git a/core/elastic/partition_test.go b/core/elastic/partition_test.go new file mode 100644 index 000000000..b07219511 --- /dev/null +++ b/core/elastic/partition_test.go @@ -0,0 +1,168 @@ +package elastic + +import ( + "net/http" + "reflect" + "strings" + "testing" + + "infini.sh/framework/core/util" +) + +func TestBuildQuantilePercents(t *testing.T) { + got := buildQuantilePercents(4) + want := []float64{25, 50, 75} + if !reflect.DeepEqual(got, want) { + t.Fatalf("unexpected percents: got %v want %v", got, want) + } +} + +func TestBuildQuantilePartitionsCreatesOpenEdgeRanges(t *testing.T) { + partitions := buildQuantilePartitions([]float64{10, 20, 30}, "value", PartitionByNumber, nil) + if len(partitions) != 2 { + t.Fatalf("unexpected partition count: %d", len(partitions)) + } + + firstRange := getMustClause(t, partitions[0].Filter)["range"].(util.MapStr)["value"].(util.MapStr) + if _, ok := firstRange["gt"]; ok { + t.Fatalf("expected first partition to have no lower bound, got %v", firstRange) + } + if got := firstRange["lte"]; got != float64(20) { + t.Fatalf("unexpected first upper bound: %v", got) + } + + secondRange := getMustClause(t, partitions[1].Filter)["range"].(util.MapStr)["value"].(util.MapStr) + if got := secondRange["gt"]; got != float64(20) { + t.Fatalf("unexpected second lower bound: %v", got) + } + if _, ok := secondRange["lte"]; ok { + t.Fatalf("expected last partition to have no upper bound, got %v", secondRange) + } +} + +func TestBuildQuantilePartitionsSinglePartitionUsesExistsFilter(t *testing.T) { + partitions := buildQuantilePartitions([]float64{5, 5}, "value", PartitionByNumber, nil) + if len(partitions) != 1 { + t.Fatalf("unexpected partition count: %d", len(partitions)) + } + + clause := getMustClause(t, partitions[0].Filter) + exists, ok := clause["exists"].(util.MapStr) + if !ok { + t.Fatalf("expected exists clause, got %v", clause) + } + if exists["field"] != "value" { + t.Fatalf("unexpected exists field: %v", exists["field"]) + } +} + +func TestBuildOpenPartitionFilterPreservesDateFormat(t *testing.T) { + upper := 1000.0 + filter := buildOpenPartitionFilter(nil, &upper, "ts", PartitionByDate, nil) + rangeFilter := getMustClause(t, filter)["range"].(util.MapStr)["ts"].(util.MapStr) + if got := rangeFilter["format"]; got != "epoch_millis" { + t.Fatalf("unexpected date format: %v", got) + } +} + +func TestBuildExactTermPartitionFilter(t *testing.T) { + filter := buildExactTermPartitionFilter("pmid-1", "pmid.keyword", nil) + termFilter := getMustClause(t, filter)["term"].(util.MapStr)["pmid.keyword"].(util.MapStr) + if got := termFilter["value"]; got != "pmid-1" { + t.Fatalf("unexpected term value: %v", got) + } +} + +func TestBuildOtherTermsPartitionFilter(t *testing.T) { + filter := buildOtherTermsPartitionFilter([]string{"a", "b"}, "pmid.keyword", nil) + boolFilter := filter["bool"].(util.MapStr) + mustNot := boolFilter["must_not"].([]interface{}) + termsFilter := mustNot[0].(util.MapStr)["terms"].(util.MapStr) + values := termsFilter["pmid.keyword"].([]string) + if !reflect.DeepEqual(values, []string{"a", "b"}) { + t.Fatalf("unexpected excluded values: %v", values) + } +} + +func TestBuildHashPartitionFilter(t *testing.T) { + filter := buildHashPartitionFilter(1, 8, "pmid.keyword", nil) + scriptFilter := getMustClause(t, filter)["script"].(util.MapStr)["script"].(util.MapStr) + if scriptFilter["lang"] != "painless" { + t.Fatalf("unexpected script language: %v", scriptFilter["lang"]) + } + source, ok := scriptFilter["source"].(string) + if !ok { + t.Fatalf("unexpected script source: %T", scriptFilter["source"]) + } + if !strings.Contains(source, "doc['pmid.keyword']") { + t.Fatalf("unexpected script source: %s", source) + } + if strings.Contains(source, "Math.floorMod") { + t.Fatalf("unexpected script source: %s", source) + } + params := scriptFilter["params"].(util.MapStr) + if params["partition_count"] != 8 || params["partition_id"] != 1 { + t.Fatalf("unexpected script params: %v", params) + } + if _, ok := params["field"]; ok { + t.Fatalf("field should not be passed as a script param: %v", params) + } +} + +func TestBuildPainlessStringLiteralEscapesSingleQuote(t *testing.T) { + got := buildPainlessStringLiteral("foo'bar") + if got != `'foo\'bar'` { + t.Fatalf("unexpected painless string literal: %s", got) + } +} + +func TestEnsurePartitionSearchResponseOKReturnsBackendReason(t *testing.T) { + err := ensurePartitionSearchResponseOK(&SearchResponse{ + ResponseBase: ResponseBase{ + StatusCode: http.StatusInternalServerError, + RawResult: &util.Result{ + Body: []byte(`{"error":{"reason":"runtime script failure"},"status":500}`), + }, + InternalError: InternalError{ + Error: &ErrorDetail{ + Reason: "runtime script failure", + }, + Status: http.StatusInternalServerError, + }, + }, + }) + if err == nil || err.Error() != "runtime script failure" { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestEnsurePartitionSearchResponseOKReturnsCausedByReason(t *testing.T) { + err := ensurePartitionSearchResponseOK(&SearchResponse{ + ResponseBase: ResponseBase{ + StatusCode: http.StatusBadRequest, + RawResult: &util.Result{ + Body: []byte(`{"error":{"root_cause":[{"reason":"compile error"}],"failed_shards":[{"reason":{"reason":"compile error","caused_by":{"reason":"static method [java.lang.Math, floorMod/2] not found"}}}],"reason":"all shards failed"},"status":400}`), + }, + }, + }) + if err == nil || err.Error() != "static method [java.lang.Math, floorMod/2] not found" { + t.Fatalf("unexpected error: %v", err) + } +} + +func getMustClause(t *testing.T, filter util.MapStr) util.MapStr { + t.Helper() + boolFilter, ok := filter["bool"].(util.MapStr) + if !ok { + t.Fatalf("expected bool filter, got %v", filter) + } + must, ok := boolFilter["must"].([]interface{}) + if !ok || len(must) == 0 { + t.Fatalf("expected must clauses, got %v", boolFilter["must"]) + } + clause, ok := must[0].(util.MapStr) + if !ok { + t.Fatalf("expected util.MapStr clause, got %T", must[0]) + } + return clause +} From 2befa0b77e908e0c07f669645e995f452b176ed6 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 16 May 2026 09:13:42 +0800 Subject: [PATCH 009/137] improve: setting delete after compress with default true --- modules/queue/disk_queue/module.go | 2 +- modules/queue/disk_queue/module_test.go | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/modules/queue/disk_queue/module.go b/modules/queue/disk_queue/module.go index deb3fe600..2383a68b4 100644 --- a/modules/queue/disk_queue/module.go +++ b/modules/queue/disk_queue/module.go @@ -271,7 +271,7 @@ func (module *DiskQueue) Setup() { PrepareFilesToRead: true, Compress: DiskCompress{ IdleThreshold: 3, - DeleteAfterCompress: false, + DeleteAfterCompress: true, NumOfFilesDecompressAhead: 3, Message: CompressConfig{ Enabled: false, diff --git a/modules/queue/disk_queue/module_test.go b/modules/queue/disk_queue/module_test.go index d17b0875b..fc643621f 100644 --- a/modules/queue/disk_queue/module_test.go +++ b/modules/queue/disk_queue/module_test.go @@ -1,6 +1,11 @@ package queue -import "testing" +import ( + "testing" + + . "infini.sh/framework/core/env" + "infini.sh/framework/core/global" +) func TestNormalizeDiskQueueConfigAppliesRobustWriteDefaults(t *testing.T) { cfg := &DiskQueueConfig{} @@ -30,3 +35,15 @@ func TestNormalizeDiskQueueConfigKeepsExplicitWriteSettings(t *testing.T) { t.Fatalf("write chan buffer should be preserved, got %d", cfg.WriteChanBuffer) } } + +func TestSetupDefaultsDeleteAfterCompress(t *testing.T) { + env1 := EmptyEnv() + global.RegisterEnv(env1) + + module := DiskQueue{} + module.Setup() + + if !module.cfg.Compress.DeleteAfterCompress { + t.Fatalf("delete_after_compress should default to true") + } +} From 9e34b492ac4059f166f7e61dc3a044fbcdf4cb46 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 16 May 2026 11:05:33 +0800 Subject: [PATCH 010/137] improve: add creator for command save --- core/elastic/common_command.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/elastic/common_command.go b/core/elastic/common_command.go index bc2cbdccf..0a4e9cd91 100644 --- a/core/elastic/common_command.go +++ b/core/elastic/common_command.go @@ -35,6 +35,7 @@ type CommonCommand struct { ID string `json:"-" index:"id"` Title string `json:"title" elastic_mapping:"title:{type:text,fields:{keyword:{type:keyword}}}"` Tag []string `json:"tag" elastic_mapping:"tag:{type:keyword}"` + Creator string `json:"creator,omitempty" elastic_mapping:"creator:{type:keyword}"` Requests []CommandRequest `json:"requests" elastic_mapping:"requests:{type:object}"` Created time.Time `json:"created,omitempty" elastic_mapping:"created:{type:date}"` } From d117e4a464c57510d00e96427e9c0af1c1259f50 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 16 May 2026 22:20:13 +0800 Subject: [PATCH 011/137] improve: scroll and bulk continue --- plugins/elastic/bulk_indexing/bulk_indexing.go | 12 ++++++++++++ plugins/elastic/bulk_indexing/bulk_indexing_test.go | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index adfa1cc11..637e6bd57 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -270,6 +270,7 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { processor.wg.Done() }() + lastDispatch := time.Now() for { if global.ShuttingDown() { @@ -306,6 +307,7 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { if global.Env().IsDebug { log.Tracef("detecting new queue: %v", v.Name) } + lastDispatch = time.Now() processor.HandleQueueConfig(v, c) } } else { @@ -317,6 +319,9 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { if processor.config.DetectIntervalInMs > 0 { time.Sleep(time.Millisecond * time.Duration(processor.config.DetectIntervalInMs)) } + if shouldQuitActiveQueueDetection(lastDispatch, time.Duration(processor.config.IdleTimeoutInSecond)*time.Second, util.MapLength(&processor.inFlightQueueConfigs)) { + return + } } }(c) } @@ -338,6 +343,13 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { return nil } +func shouldQuitActiveQueueDetection(lastDispatch time.Time, idleDuration time.Duration, inflight int) bool { + if idleDuration <= 0 { + return false + } + return inflight == 0 && time.Since(lastDispatch) > idleDuration +} + const queueHandleSingleton = "queue_handler_singleton" func (processor *BulkIndexingProcessor) HandleQueueConfig(v *queue.QueueConfig, parentContext *pipeline.Context) { diff --git a/plugins/elastic/bulk_indexing/bulk_indexing_test.go b/plugins/elastic/bulk_indexing/bulk_indexing_test.go index aa43bd8f2..f3cfa468a 100644 --- a/plugins/elastic/bulk_indexing/bulk_indexing_test.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing_test.go @@ -33,6 +33,7 @@ import ( "github.com/stretchr/testify/assert" "sync" "testing" + "time" ) func TestXXHash(t *testing.T) { @@ -154,3 +155,9 @@ func TestIsIgnorableAcquireConsumerError(t *testing.T) { assert.False(t, isIgnorableAcquireConsumerError(stdErrors.New("some other error"))) assert.False(t, isIgnorableAcquireConsumerError(nil)) } + +func TestShouldQuitActiveQueueDetection(t *testing.T) { + assert.False(t, shouldQuitActiveQueueDetection(time.Now(), 5*time.Second, 0)) + assert.False(t, shouldQuitActiveQueueDetection(time.Now().Add(-10*time.Second), 5*time.Second, 1)) + assert.True(t, shouldQuitActiveQueueDetection(time.Now().Add(-10*time.Second), 5*time.Second, 0)) +} From ee806e0f05b0a9f37883868c5faeafeeafdad3ec Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 17 May 2026 07:36:31 +0800 Subject: [PATCH 012/137] fix: config callback at first then pipeline restart --- core/config/fs_watcher.go | 69 ++++++++++++++++------------------ core/config/fs_watcher_test.go | 47 +++++++++++++++++++++++ 2 files changed, 80 insertions(+), 36 deletions(-) create mode 100644 core/config/fs_watcher_test.go diff --git a/core/config/fs_watcher.go b/core/config/fs_watcher.go index 45706c402..a81408d36 100644 --- a/core/config/fs_watcher.go +++ b/core/config/fs_watcher.go @@ -62,6 +62,34 @@ func loadConfigFile(file string) *Config { return nil } +func dispatchConfigChangeEvent(ev fsnotify.Event, watcherCallbacks []CallbackFunc) { + for _, v := range watcherCallbacks { + v(ev.Name, ev.Op) + } + + cfg := loadConfigFile(ev.Name) + if cfg != nil { + for k, v := range sectionCallbacks { + if cfg.HasField(k) { + currentCfg, err := cfg.Child(k, -1) + if err != nil { + log.Error(err) + continue + } + previousCfg, _ := latestConfig[k] + for _, f := range v { + f(previousCfg, currentCfg) + } + latestConfig[k] = currentCfg + } + } + } + + for _, v := range configCallbacks { + v(ev) + } +} + var validExtensions = []string{".yml", ".yaml", ".tpl"} func SetValidExtension(v []string) { @@ -153,40 +181,7 @@ func AddPathToWatch(path string, callback CallbackFunc) { time.Sleep(2 * time.Second) log.Trace("2 seconds out, on:", ev.String()) - // AddPathToWatch - - for _, v := range watcher.callbacks { - v(ev.Name, ev.Op) - } - - // NotifyOnConfigChange - - for _, v := range configCallbacks { - v(ev) - } - - // NotifyOnConfigSectionChange - - cfg := loadConfigFile(ev.Name) - if cfg == nil { - continue - } - - for k, v := range sectionCallbacks { - if cfg.HasField(k) { - currentCfg, err := cfg.Child(k, -1) - if err != nil { - log.Error(err) - continue - } - // diff config - previousCfg, _ := latestConfig[k] - for _, f := range v { - f(previousCfg, currentCfg) - } - latestConfig[k] = currentCfg - } - } + dispatchConfigChangeEvent(ev, watcher.callbacks) } }() }) @@ -259,7 +254,8 @@ var configCallbacks = []func(fsnotify.Event){} var cfgLocker = sync.RWMutex{} // NotifyOnConfigSectionChange will trigger callback when any configuration file change detected and -// configKey present in the changed file +// configKey present in the changed file. Section callbacks run before generic NotifyOnConfigChange +// callbacks so section-scoped state can be refreshed before dependent consumers reload. func NotifyOnConfigSectionChange(configKey string, f func(pCfg, cCfg *Config)) { cfgLocker.Lock() defer cfgLocker.Unlock() @@ -273,7 +269,8 @@ func NotifyOnConfigSectionChange(configKey string, f func(pCfg, cCfg *Config)) { sectionCallbacks[configKey] = v } -// NotifyOnConfigChange will trigger callback when any configuration file change detected +// NotifyOnConfigChange will trigger callback when any configuration file change detected, after any +// matching NotifyOnConfigSectionChange callbacks for the same event have run. func NotifyOnConfigChange(f func(fsnotify.Event)) { cfgLocker.Lock() defer cfgLocker.Unlock() diff --git a/core/config/fs_watcher_test.go b/core/config/fs_watcher_test.go new file mode 100644 index 000000000..29d4559d4 --- /dev/null +++ b/core/config/fs_watcher_test.go @@ -0,0 +1,47 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/fsnotify/fsnotify" +) + +func TestDispatchConfigChangeEventRunsSectionCallbacksBeforeGenericCallbacks(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "generated_metrics_tasks.yml") + content := []byte("elasticsearch:\n - id: \"cluster-1\"\n name: \"cluster-1\"\n enabled: true\n endpoint: \"http://127.0.0.1:9200\"\n") + if err := os.WriteFile(file, content, 0o644); err != nil { + t.Fatalf("write config file: %v", err) + } + + previousSections := sectionCallbacks + previousConfigs := configCallbacks + previousLatest := latestConfig + sectionCallbacks = map[string][]func(pCfg, cCfg *Config){} + configCallbacks = nil + latestConfig = map[string]*Config{} + t.Cleanup(func() { + sectionCallbacks = previousSections + configCallbacks = previousConfigs + latestConfig = previousLatest + }) + + var order []string + NotifyOnConfigSectionChange("elasticsearch", func(pCfg, cCfg *Config) { + order = append(order, "section") + }) + NotifyOnConfigChange(func(ev fsnotify.Event) { + order = append(order, "generic") + }) + + dispatchConfigChangeEvent(fsnotify.Event{Name: file, Op: fsnotify.Write}, nil) + + if len(order) != 2 { + t.Fatalf("expected 2 callbacks, got %d (%v)", len(order), order) + } + if order[0] != "section" || order[1] != "generic" { + t.Fatalf("expected section callback before generic callback, got %v", order) + } +} From c3b0f5d91cf2be739fab6c7c2badd21e0ac3cb7c Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 17 May 2026 11:05:40 +0800 Subject: [PATCH 013/137] fix: init metadata by first register cluster --- core/elastic/domain_actions.go | 8 ++++++- core/elastic/domain_actions_test.go | 35 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 core/elastic/domain_actions_test.go diff --git a/core/elastic/domain_actions.go b/core/elastic/domain_actions.go index d40bceadf..90e84f388 100644 --- a/core/elastic/domain_actions.go +++ b/core/elastic/domain_actions.go @@ -99,8 +99,14 @@ func RegisterInstance(cfg ElasticsearchConfig, handler API) { UpdateClient(cfg, handler) UpdateConfig(cfg) + meta := GetMetadata(cfg.ID) + if meta == nil { + InitMetadata(&cfg, false) + return + } + if exists && oldCfg != nil { - InitMetadata(&cfg, true) + InitMetadata(&cfg, meta.IsAvailable()) } } diff --git a/core/elastic/domain_actions_test.go b/core/elastic/domain_actions_test.go new file mode 100644 index 000000000..1a2c05a56 --- /dev/null +++ b/core/elastic/domain_actions_test.go @@ -0,0 +1,35 @@ +package elastic + +import ( + "testing" + + "infini.sh/framework/core/orm" +) + +func TestRegisterInstanceInitializesMetadataOnFirstRegistration(t *testing.T) { + cfg := ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: "test-first-sync"}, + Name: "test-first-sync", + Enabled: true, + ClusterUUID: "cluster-uuid-1", + } + + t.Cleanup(func() { + cfgs.Delete(cfg.ID) + apis.Delete(cfg.ID) + metas.Delete(cfg.ID) + }) + + RegisterInstance(cfg, nil) + + meta := GetMetadata(cfg.ID) + if meta == nil { + t.Fatalf("expected metadata to be initialized for %s", cfg.ID) + } + if meta.Config == nil { + t.Fatalf("expected metadata config to be initialized for %s", cfg.ID) + } + if meta.Config.ClusterUUID != cfg.ClusterUUID { + t.Fatalf("expected cluster uuid %q, got %q", cfg.ClusterUUID, meta.Config.ClusterUUID) + } +} From 64f933073c1a143ca16de61a8caa5b965dbf02f2 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 17 May 2026 11:42:57 +0800 Subject: [PATCH 014/137] fix: collect mode change with metrics for pipeline --- modules/metrics/elastic/elasticsearch.go | 22 ++++++++++- modules/metrics/elastic/elasticsearch_test.go | 39 +++++++++++++++++++ modules/metrics/metrics.go | 3 +- 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 modules/metrics/elastic/elasticsearch_test.go diff --git a/modules/metrics/elastic/elasticsearch.go b/modules/metrics/elastic/elasticsearch.go index 54d9c8357..65efcdcb2 100644 --- a/modules/metrics/elastic/elasticsearch.go +++ b/modules/metrics/elastic/elasticsearch.go @@ -127,6 +127,24 @@ func validateMonitorConfig(monitorConfig *elastic.TaskConfig) { } } +func (m *ElasticsearchMetric) shouldCollectMetrics(v *elastic.ElasticsearchMetadata) bool { + if v == nil || v.Config == nil { + return false + } + if !v.Config.Monitored || !v.Config.Enabled { + return false + } + if !m.IsAgentMode && v.Config.MetricCollectionMode == elastic.ModeAgent { + log.Debugf("cluster [%v] is in agent mode, skip console-side metric collection", v.Config.Name) + return false + } + if m.IsAgentMode && v.Config.MetricCollectionMode == elastic.ModeAgentless { + log.Debugf("cluster [%v] is in agentless mode, skip agent-side metric collection", v.Config.Name) + return false + } + return true +} + func (m *ElasticsearchMetric) Collect() error { if !m.Enabled { return nil @@ -192,8 +210,8 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea m.RemoveTask(taskID) } } - if !v.Config.Monitored || !v.Config.Enabled { - log.Debugf("cluster [%v] NOT (enabled[%v] or monitored[%v] or not available[%v]), skip collect", v.Config.Name, v.Config.Enabled, v.Config.Monitored, v.IsAvailable()) + if !m.shouldCollectMetrics(v) { + log.Debugf("cluster [%v] NOT eligible for metrics collection (enabled[%v], monitored[%v], mode[%v], available[%v]), skip collect", v.Config.Name, v.Config.Enabled, v.Config.Monitored, v.Config.MetricCollectionMode, v.IsAvailable()) return true } if global.Env().IsDebug { diff --git a/modules/metrics/elastic/elasticsearch_test.go b/modules/metrics/elastic/elasticsearch_test.go new file mode 100644 index 000000000..a023fa8b7 --- /dev/null +++ b/modules/metrics/elastic/elasticsearch_test.go @@ -0,0 +1,39 @@ +package elastic + +import ( + "testing" + + coreelastic "infini.sh/framework/core/elastic" +) + +func TestShouldCollectMetricsSkipsConsoleCollectorInAgentMode(t *testing.T) { + collector := &ElasticsearchMetric{} + meta := &coreelastic.ElasticsearchMetadata{ + Config: &coreelastic.ElasticsearchConfig{ + Name: "agent-cluster", + Enabled: true, + Monitored: true, + MetricCollectionMode: coreelastic.ModeAgent, + }, + } + + if collector.shouldCollectMetrics(meta) { + t.Fatal("expected console-side collector to skip clusters in agent mode") + } +} + +func TestShouldCollectMetricsAllowsConsoleCollectorInAgentlessMode(t *testing.T) { + collector := &ElasticsearchMetric{} + meta := &coreelastic.ElasticsearchMetadata{ + Config: &coreelastic.ElasticsearchConfig{ + Name: "agentless-cluster", + Enabled: true, + Monitored: true, + MetricCollectionMode: coreelastic.ModeAgentless, + }, + } + + if !collector.shouldCollectMetrics(meta) { + t.Fatal("expected console-side collector to run for agentless clusters") + } +} diff --git a/modules/metrics/metrics.go b/modules/metrics/metrics.go index 5fae56cc6..148f4e556 100755 --- a/modules/metrics/metrics.go +++ b/modules/metrics/metrics.go @@ -115,7 +115,8 @@ func (module *MetricsModule) Setup() { // check other conditions hasChanged = meta.IsAvailable() != oldMeta.IsAvailable() || meta.Config.Enabled != oldMeta.Config.Enabled || - meta.Config.Monitored != oldMeta.Config.Monitored + meta.Config.Monitored != oldMeta.Config.Monitored || + meta.Config.MetricCollectionMode != oldMeta.Config.MetricCollectionMode } if !hasChanged { return From b919298633874aec16d15020a82d586e173d2028 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 17 May 2026 12:20:46 +0800 Subject: [PATCH 015/137] fix: collect mode change with metrics for pipeline --- modules/elastic/adapter/ver.go | 8 ++++++++ modules/elastic/adapter/ver_test.go | 30 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 modules/elastic/adapter/ver_test.go diff --git a/modules/elastic/adapter/ver.go b/modules/elastic/adapter/ver.go index 1e3120cb5..a02119217 100755 --- a/modules/elastic/adapter/ver.go +++ b/modules/elastic/adapter/ver.go @@ -169,6 +169,14 @@ func RequestTimeout(ctx *elastic.APIContext, method, url string, body []byte, me func GetClusterUUID(clusterID string) (string, error) { meta := elastic.GetMetadata(clusterID) + if meta == nil { + if cfg := elastic.GetConfigNoPanic(clusterID); cfg != nil { + if cfg.ClusterUUID != "" { + return cfg.ClusterUUID, nil + } + meta = elastic.GetOrInitMetadata(cfg) + } + } if meta == nil { return "", fmt.Errorf("metadata can not be mepty") } diff --git a/modules/elastic/adapter/ver_test.go b/modules/elastic/adapter/ver_test.go new file mode 100644 index 000000000..db5808e55 --- /dev/null +++ b/modules/elastic/adapter/ver_test.go @@ -0,0 +1,30 @@ +package adapter + +import ( + "testing" + + "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" +) + +func TestGetClusterUUIDFallsBackToConfigWhenMetadataMissing(t *testing.T) { + cfg := elastic.ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: "test-cluster-uuid-fallback"}, + Name: "test-cluster-uuid-fallback", + ClusterUUID: "cluster-uuid-fallback", + } + + t.Cleanup(func() { + elastic.RemoveInstance(cfg.ID) + }) + + elastic.UpdateConfig(cfg) + + clusterUUID, err := GetClusterUUID(cfg.ID) + if err != nil { + t.Fatalf("expected cluster uuid from config fallback, got error: %v", err) + } + if clusterUUID != cfg.ClusterUUID { + t.Fatalf("expected cluster uuid %q, got %q", cfg.ClusterUUID, clusterUUID) + } +} From 74613279f6bbacd93dda3c1aebf197217cdf5a81 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 17 May 2026 22:07:24 +0800 Subject: [PATCH 016/137] fix: agent mode with metrics collect --- modules/metrics/elastic/elasticsearch.go | 33 ++++++++++---- modules/metrics/elastic/elasticsearch_test.go | 44 +++++++++++++++++-- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/modules/metrics/elastic/elasticsearch.go b/modules/metrics/elastic/elasticsearch.go index 65efcdcb2..e3dae62b5 100644 --- a/modules/metrics/elastic/elasticsearch.go +++ b/modules/metrics/elastic/elasticsearch.go @@ -134,10 +134,6 @@ func (m *ElasticsearchMetric) shouldCollectMetrics(v *elastic.ElasticsearchMetad if !v.Config.Monitored || !v.Config.Enabled { return false } - if !m.IsAgentMode && v.Config.MetricCollectionMode == elastic.ModeAgent { - log.Debugf("cluster [%v] is in agent mode, skip console-side metric collection", v.Config.Name) - return false - } if m.IsAgentMode && v.Config.MetricCollectionMode == elastic.ModeAgentless { log.Debugf("cluster [%v] is in agentless mode, skip agent-side metric collection", v.Config.Name) return false @@ -145,6 +141,21 @@ func (m *ElasticsearchMetric) shouldCollectMetrics(v *elastic.ElasticsearchMetad return true } +func (m *ElasticsearchMetric) shouldCollectClusterLevelMetrics(v *elastic.ElasticsearchMetadata) bool { + return m.shouldCollectMetrics(v) +} + +func (m *ElasticsearchMetric) shouldCollectNodeAndIndexMetrics(v *elastic.ElasticsearchMetadata) bool { + if !m.shouldCollectMetrics(v) { + return false + } + if !m.IsAgentMode && v.Config.MetricCollectionMode == elastic.ModeAgent { + log.Debugf("cluster [%v] is in agent mode, skip console-side node/index metric collection", v.Config.Name) + return false + } + return true +} + func (m *ElasticsearchMetric) Collect() error { if !m.Enabled { return nil @@ -220,7 +231,13 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea var err error monitorConfigs := getMonitorConfigs(v) - if m.ClusterHealth && monitorConfigs.ClusterHealth.Enabled { + clusterLevelEnabled := m.shouldCollectClusterLevelMetrics(v) + nodeAndIndexEnabled := m.shouldCollectNodeAndIndexMetrics(v) + if !clusterLevelEnabled && !nodeAndIndexEnabled { + log.Debugf("cluster [%v] has no eligible metric collectors (mode[%v], agent_mode[%v])", v.Config.Name, v.Config.MetricCollectionMode, m.IsAgentMode) + return true + } + if clusterLevelEnabled && m.ClusterHealth && monitorConfigs.ClusterHealth.Enabled { log.Debugf("collect cluster health: %s, endpoint: %s", k, v.Config.GetAnyEndpoint()) var clusterHealthMetricTask = task.ScheduleTask{ ID: clusterHealthTaskID, @@ -244,7 +261,7 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea } //cluster stats - if m.ClusterStats && monitorConfigs.ClusterStats.Enabled { + if clusterLevelEnabled && m.ClusterStats && monitorConfigs.ClusterStats.Enabled { log.Debugf("collect cluster state: %s, endpoint: %s", k, v.Config.GetAnyEndpoint()) var clusterStatsMetricTask = task.ScheduleTask{ ID: clusterStatsTaskID, @@ -268,7 +285,7 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea } //nodes stats - if m.NodeStats && monitorConfigs.NodeStats.Enabled { + if nodeAndIndexEnabled && m.NodeStats && monitorConfigs.NodeStats.Enabled { var nodeStatsMetricTask = task.ScheduleTask{ ID: nodeStatsTaskID, Description: fmt.Sprintf("monitoring node stats metric for cluster %s", k), @@ -344,7 +361,7 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea } //indices stats - if (m.AllIndexStats || m.IndexStats) && monitorConfigs.IndexStats.Enabled { + if nodeAndIndexEnabled && (m.AllIndexStats || m.IndexStats) && monitorConfigs.IndexStats.Enabled { var indexStatsMetricTask = task.ScheduleTask{ ID: indexStatsTaskID, Description: fmt.Sprintf("monitoring index stats metric for cluster %s", k), diff --git a/modules/metrics/elastic/elasticsearch_test.go b/modules/metrics/elastic/elasticsearch_test.go index a023fa8b7..eead81c34 100644 --- a/modules/metrics/elastic/elasticsearch_test.go +++ b/modules/metrics/elastic/elasticsearch_test.go @@ -6,7 +6,7 @@ import ( coreelastic "infini.sh/framework/core/elastic" ) -func TestShouldCollectMetricsSkipsConsoleCollectorInAgentMode(t *testing.T) { +func TestShouldCollectMetricsAllowsConsoleCollectorInAgentModeForClusterLevelMetrics(t *testing.T) { collector := &ElasticsearchMetric{} meta := &coreelastic.ElasticsearchMetadata{ Config: &coreelastic.ElasticsearchConfig{ @@ -17,8 +17,11 @@ func TestShouldCollectMetricsSkipsConsoleCollectorInAgentMode(t *testing.T) { }, } - if collector.shouldCollectMetrics(meta) { - t.Fatal("expected console-side collector to skip clusters in agent mode") + if !collector.shouldCollectMetrics(meta) { + t.Fatal("expected console-side collector to keep cluster-level metrics in agent mode") + } + if collector.shouldCollectNodeAndIndexMetrics(meta) { + t.Fatal("expected console-side collector to skip node/index metrics in agent mode") } } @@ -36,4 +39,39 @@ func TestShouldCollectMetricsAllowsConsoleCollectorInAgentlessMode(t *testing.T) if !collector.shouldCollectMetrics(meta) { t.Fatal("expected console-side collector to run for agentless clusters") } + if !collector.shouldCollectNodeAndIndexMetrics(meta) { + t.Fatal("expected console-side collector to run node/index metrics for agentless clusters") + } +} + +func TestShouldCollectMetricsSkipsAgentCollectorInAgentlessMode(t *testing.T) { + collector := &ElasticsearchMetric{IsAgentMode: true} + meta := &coreelastic.ElasticsearchMetadata{ + Config: &coreelastic.ElasticsearchConfig{ + Name: "agentless-cluster", + Enabled: true, + Monitored: true, + MetricCollectionMode: coreelastic.ModeAgentless, + }, + } + + if collector.shouldCollectMetrics(meta) { + t.Fatal("expected agent-side collector to skip agentless clusters") + } +} + +func TestShouldCollectMetricsAllowsAgentCollectorInAgentMode(t *testing.T) { + collector := &ElasticsearchMetric{IsAgentMode: true} + meta := &coreelastic.ElasticsearchMetadata{ + Config: &coreelastic.ElasticsearchConfig{ + Name: "agent-cluster", + Enabled: true, + Monitored: true, + MetricCollectionMode: coreelastic.ModeAgent, + }, + } + + if !collector.shouldCollectMetrics(meta) { + t.Fatal("expected agent-side collector to run for agent mode clusters") + } } From d786b499e2a687b5c3a5ffa346929687aa793126 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 07:27:49 +0800 Subject: [PATCH 017/137] fix: registory for gateway --- core/orm/registry.go | 16 ++++++++++++++++ core/orm/registry_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 core/orm/registry_test.go diff --git a/core/orm/registry.go b/core/orm/registry.go index 292b42e36..1cf2a1f9c 100644 --- a/core/orm/registry.go +++ b/core/orm/registry.go @@ -10,6 +10,11 @@ import ( var registeredSchemas = []util.KeyValue{} +func schemaRegistrationKey(t interface{}) string { + pkg, typeName := util.GetTypeAndPackageName(t, true) + return pkg + "-" + typeName +} + func MustRegisterSchemaWithIndexName(t interface{}, index string) { err := RegisterSchemaWithIndexName(t, index) if err != nil { @@ -18,6 +23,17 @@ func MustRegisterSchemaWithIndexName(t interface{}, index string) { } func RegisterSchemaWithIndexName(t interface{}, index string) error { + newKey := schemaRegistrationKey(t) + for _, registered := range registeredSchemas { + if registered.Key != index { + continue + } + existingKey := schemaRegistrationKey(registered.Payload) + if existingKey == newKey { + return nil + } + return errors.Errorf("schema index [%s] already registered by [%s]", index, existingKey) + } registeredSchemas = append(registeredSchemas, util.KeyValue{Key: index, Payload: t}) return nil } diff --git a/core/orm/registry_test.go b/core/orm/registry_test.go new file mode 100644 index 000000000..b74dc34dd --- /dev/null +++ b/core/orm/registry_test.go @@ -0,0 +1,39 @@ +package orm + +import "testing" + +type testSchemaAlpha struct{} +type testSchemaBeta struct{} + +func TestRegisterSchemaWithIndexNameDeduplicatesSameSchema(t *testing.T) { + original := registeredSchemas + registeredSchemas = nil + t.Cleanup(func() { + registeredSchemas = original + }) + + if err := RegisterSchemaWithIndexName(testSchemaAlpha{}, "test-index"); err != nil { + t.Fatalf("expected first registration to succeed, got %v", err) + } + if err := RegisterSchemaWithIndexName(&testSchemaAlpha{}, "test-index"); err != nil { + t.Fatalf("expected duplicate registration to be ignored, got %v", err) + } + if len(registeredSchemas) != 1 { + t.Fatalf("expected exactly one registered schema, got %d", len(registeredSchemas)) + } +} + +func TestRegisterSchemaWithIndexNameRejectsDifferentSchemaForSameIndex(t *testing.T) { + original := registeredSchemas + registeredSchemas = nil + t.Cleanup(func() { + registeredSchemas = original + }) + + if err := RegisterSchemaWithIndexName(testSchemaAlpha{}, "test-index"); err != nil { + t.Fatalf("expected first registration to succeed, got %v", err) + } + if err := RegisterSchemaWithIndexName(testSchemaBeta{}, "test-index"); err == nil { + t.Fatal("expected conflicting registration to fail") + } +} From 031f4811bc18051da622cf80437cf469bff7507d Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 10:32:11 +0800 Subject: [PATCH 018/137] improve: migration with api context --- core/pipeline/context.go | 42 +++++++++++++++++++- core/pipeline/context_result_test.go | 59 ++++++++++++++++++++++++++++ modules/pipeline/api.go | 17 +++++--- modules/pipeline/model.go | 21 ++++++---- 4 files changed, 125 insertions(+), 14 deletions(-) create mode 100644 core/pipeline/context_result_test.go diff --git a/core/pipeline/context.go b/core/pipeline/context.go index 62008306a..8936789e4 100755 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -289,6 +289,20 @@ func (ctx *Context) Errors() []error { return ctx.processErrs } +func (ctx *Context) GetResultState() RunningState { + ctx.stateLock.Lock() + defer ctx.stateLock.Unlock() + + return ctx.getResultStateLocked() +} + +func (ctx *Context) GetResultError() string { + ctx.stateLock.Lock() + defer ctx.stateLock.Unlock() + + return formatPipelineResultError(ctx.exitErr, ctx.processErrs) +} + // Pause will pause the pipeline running loop until Resume called func (ctx *Context) Pause() { ctx.stateLock.Lock() @@ -368,6 +382,30 @@ func (ctx *Context) setRunningState(newState RunningState) { } } +func (ctx *Context) getResultStateLocked() RunningState { + switch ctx.runningState { + case FINISHED, FAILED: + return ctx.runningState + case STOPPED: + if ctx.endTime == nil { + return STOPPED + } + if ctx.exitErr != nil || len(ctx.processErrs) > 0 { + return FAILED + } + return FINISHED + default: + return "" + } +} + +func formatPipelineResultError(exitErr error, processErrs []error) string { + if exitErr == nil && len(processErrs) == 0 { + return "" + } + return fmt.Sprintf("exit: %v, process: %v", exitErr, processErrs) +} + func (ctx *Context) pushPipelineLog() { if global.Env().IsDebug { log.Info("received pipeline state change, id: ", ctx.Config.Name, ", state: ", ctx.runningState) @@ -397,8 +435,8 @@ func (ctx *Context) pushPipelineLog() { result := util.MapStr{ "success": ctx.exitErr == nil, } - if ctx.exitErr != nil || len(ctx.processErrs) > 0 { - result["error"] = fmt.Sprintf("exit: %v, process: %v", ctx.exitErr, ctx.processErrs) + if errMsg := formatPipelineResultError(ctx.exitErr, ctx.processErrs); errMsg != "" { + result["error"] = errMsg } payload["result"] = result } diff --git a/core/pipeline/context_result_test.go b/core/pipeline/context_result_test.go new file mode 100644 index 000000000..addb8d4b5 --- /dev/null +++ b/core/pipeline/context_result_test.go @@ -0,0 +1,59 @@ +package pipeline + +import ( + "errors" + "testing" +) + +func TestGetResultStateReturnsFinishedAfterStoppedCompletedRun(t *testing.T) { + ctx := AcquireContext(PipelineConfigV2{}) + ctx.Started() + ctx.Finished() + ctx.Stopped() + + if got := ctx.GetResultState(); got != FINISHED { + t.Fatalf("expected FINISHED result state, got %q", got) + } + if got := ctx.GetResultError(); got != "" { + t.Fatalf("expected empty result error, got %q", got) + } +} + +func TestGetResultStateReturnsFailedAfterStoppedFailedRun(t *testing.T) { + ctx := AcquireContext(PipelineConfigV2{}) + ctx.Started() + ctx.Failed(errors.New("boom")) + ctx.Stopped() + + if got := ctx.GetResultState(); got != FAILED { + t.Fatalf("expected FAILED result state, got %q", got) + } + if got := ctx.GetResultError(); got == "" { + t.Fatal("expected result error for failed run") + } +} + +func TestGetResultStateReturnsStoppedForManualStop(t *testing.T) { + ctx := AcquireContext(PipelineConfigV2{}) + ctx.Started() + ctx.Stopping() + ctx.Stopped() + + if got := ctx.GetResultState(); got != STOPPED { + t.Fatalf("expected STOPPED result state, got %q", got) + } + if got := ctx.GetResultError(); got != "" { + t.Fatalf("expected empty result error, got %q", got) + } +} + +func TestGetResultErrorIncludesProcessErrors(t *testing.T) { + ctx := AcquireContext(PipelineConfigV2{}) + ctx.Started() + ctx.RecordError(errors.New("slice failed")) + ctx.Finished() + + if got := ctx.GetResultError(); got == "" { + t.Fatal("expected process error to be surfaced") + } +} diff --git a/modules/pipeline/api.go b/modules/pipeline/api.go index d34346ec0..1efcfd3bc 100644 --- a/modules/pipeline/api.go +++ b/modules/pipeline/api.go @@ -80,11 +80,18 @@ func (module *PipeModule) getPipelineStatus(id string, config string, processor return nil } ret := &PipelineStatus{ - State: c1.GetRunningState(), - CreateTime: c1.GetCreateTime(), - StartTime: c1.GetStartTime(), - EndTime: c1.GetEndTime(), - Context: c1.CloneData(), + State: c1.GetRunningState(), + LastRunState: c1.GetResultState(), + CreateTime: c1.GetCreateTime(), + StartTime: c1.GetStartTime(), + EndTime: c1.GetEndTime(), + Context: c1.CloneData(), + } + if ret.LastRunState == pipeline.FINISHED || ret.LastRunState == pipeline.FAILED { + ret.Result = &PipelineResult{ + Success: c1.GetResultError() == "", + Error: c1.GetResultError(), + } } if config != "false" { v1, ok := module.configs.Load(id) diff --git a/modules/pipeline/model.go b/modules/pipeline/model.go index dcb79679d..bdbf305d7 100644 --- a/modules/pipeline/model.go +++ b/modules/pipeline/model.go @@ -31,11 +31,18 @@ import ( ) type PipelineStatus struct { - State pipeline.RunningState `json:"state"` - CreateTime time.Time `json:"create_time"` - StartTime *time.Time `json:"start_time"` - EndTime *time.Time `json:"end_time"` - Context util.MapStr `json:"context"` - Config *pipeline.PipelineConfigV2 `json:"config"` - Processors []map[string]interface{} `json:"processor"` + State pipeline.RunningState `json:"state"` + LastRunState pipeline.RunningState `json:"last_run_state,omitempty"` + CreateTime time.Time `json:"create_time"` + StartTime *time.Time `json:"start_time"` + EndTime *time.Time `json:"end_time"` + Context util.MapStr `json:"context"` + Result *PipelineResult `json:"result,omitempty"` + Config *pipeline.PipelineConfigV2 `json:"config"` + Processors []map[string]interface{} `json:"processor"` +} + +type PipelineResult struct { + Success bool `json:"success"` + Error string `json:"error,omitempty"` } From 86815e5541adc99eac65d793750fb11a57839e4d Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 14:09:28 +0800 Subject: [PATCH 019/137] fix: console setup failed without initialized --- modules/elastic/module.go | 46 +++++++++++++++----- modules/elastic/module_test.go | 78 +++++++++++++++++----------------- 2 files changed, 75 insertions(+), 49 deletions(-) diff --git a/modules/elastic/module.go b/modules/elastic/module.go index a1f11fb84..56ad1ca42 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -112,10 +112,23 @@ func loadFileBasedElasticConfig() []elastic.ElasticsearchConfig { return configs } +func lookupSystemElasticsearchID() (string, bool) { + value := global.Lookup(elastic.GlobalSystemElasticsearchID) + systemID, ok := value.(string) + if !ok || systemID == "" { + return "", false + } + return systemID, true +} + func loadESBasedElasticConfig() []elastic.ElasticsearchConfig { configs := []elastic.ElasticsearchConfig{} + systemID, ok := lookupSystemElasticsearchID() + if !ok { + return configs + } query := elastic.SearchRequest{From: 0, Size: 1000} //TODO handle clusters beyond 1000 - esClient := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) + esClient := elastic.GetClient(systemID) result, err := esClient.Search(orm.GetIndexName(elastic.ElasticsearchConfig{}), &query) if err != nil { log.Error(err) @@ -394,20 +407,29 @@ func InitSchema() { var ormInited bool func (module *ElasticModule) Start() error { + systemID, hasSystemCluster := lookupSystemElasticsearchID() if moduleConfig.ORMConfig.Enabled { - client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) - handler := ElasticORM{Client: client, Config: moduleConfig.ORMConfig} - orm.Register("elastic", &handler) + if !hasSystemCluster { + log.Warn("skip elastic ORM initialization, system cluster is not available") + } else { + client := elastic.GetClient(systemID) + handler := ElasticORM{Client: client, Config: moduleConfig.ORMConfig} + orm.Register("elastic", &handler) + } } if moduleConfig.StoreConfig.Enabled { - client := elastic.GetClient(global.MustLookupString(elastic.GlobalSystemElasticsearchID)) - module.storeHandler = &ElasticStore{Client: client, Config: moduleConfig.StoreConfig} - kv.Register("elastic", module.storeHandler) + if !hasSystemCluster { + log.Warn("skip elastic store initialization, system cluster is not available") + } else { + client := elastic.GetClient(systemID) + module.storeHandler = &ElasticStore{Client: client, Config: moduleConfig.StoreConfig} + kv.Register("elastic", module.storeHandler) + } } - if moduleConfig.ORMConfig.Enabled { + if moduleConfig.ORMConfig.Enabled && hasSystemCluster { if !ormInited { //init template InitTemplate(false) @@ -418,8 +440,12 @@ func (module *ElasticModule) Start() error { } if moduleConfig.RemoteConfigEnabled { - m := loadESBasedElasticConfig() - initElasticInstances(m, elastic.ElasticsearchConfigSourceElasticsearch) + if !hasSystemCluster { + log.Warn("skip remote elastic config loading, system cluster is not available") + } else { + m := loadESBasedElasticConfig() + initElasticInstances(m, elastic.ElasticsearchConfigSourceElasticsearch) + } } if module.storeHandler != nil { diff --git a/modules/elastic/module_test.go b/modules/elastic/module_test.go index 3accbf667..e88f7e39c 100644 --- a/modules/elastic/module_test.go +++ b/modules/elastic/module_test.go @@ -1,48 +1,48 @@ -// Copyright (C) INFINI Labs & INFINI LIMITED. -// -// The INFINI Framework is offered under the GNU Affero General Public License v3.0 -// and as commercial software. -// -// For commercial licensing, contact us at: -// - Website: infinilabs.com -// - Email: hello@infini.ltd -// -// Open Source licensed under AGPL V3: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - package elastic import ( - "fmt" - "github.com/buger/jsonparser" - "infini.sh/framework/core/util" "testing" + + coreElastic "infini.sh/framework/core/elastic" + "infini.sh/framework/core/global" ) -func TestV7GetClusterStates(t *testing.T) { - str := "{ \"_nodes\": { \"total\": 1, \"successful\": 1, \"failed\": 0 }, \"cluster_name\": \"es-v700\", \"cluster_uuid\": \"7NtDffC3RzGChhoOmgySig\", \"timestamp\": 1629611578327, \"status\": \"green\", \"indices\": { \"count\": 0, \"shards\": {}, \"docs\": { \"count\": 0, \"deleted\": 0 }, \"store\": { \"size_in_bytes\": 0 }, \"fielddata\": { \"memory_size_in_bytes\": 0, \"evictions\": 0 }, \"query_cache\": { \"memory_size_in_bytes\": 0, \"total_count\": 0, \"hit_count\": 0, \"miss_count\": 0, \"cache_size\": 0, \"cache_count\": 0, \"evictions\": 0 }, \"completion\": { \"size_in_bytes\": 0 }, \"segments\": { \"count\": 0, \"memory_in_bytes\": 0, \"terms_memory_in_bytes\": 0, \"stored_fields_memory_in_bytes\": 0, \"term_vectors_memory_in_bytes\": 0, \"norms_memory_in_bytes\": 0, \"points_memory_in_bytes\": 0, \"doc_values_memory_in_bytes\": 0, \"index_writer_memory_in_bytes\": 0, \"version_map_memory_in_bytes\": 0, \"fixed_bit_set_memory_in_bytes\": 0, \"max_unsafe_auto_id_timestamp\": -9223372036854776000, \"file_sizes\": {} } }, \"nodes\": { \"count\": { \"total\": 1, \"data\": 1, \"coordinating_only\": 0, \"master\": 1, \"ingest\": 1 }, \"versions\": [ \"7.0.0\" ], \"os\": { \"available_processors\": 24, \"allocated_processors\": 24, \"names\": [ { \"name\": \"Windows 10\", \"count\": 1 } ], \"pretty_names\": [ { \"pretty_name\": \"Windows 10\", \"count\": 1 } ], \"mem\": { \"total_in_bytes\": 137121308672, \"free_in_bytes\": 114813546496, \"used_in_bytes\": 22307762176, \"free_percent\": 84, \"used_percent\": 16 } }, \"process\": { \"cpu\": { \"percent\": 0 }, \"open_file_descriptors\": { \"min\": -1, \"max\": -1, \"avg\": 0 } }, \"jvm\": { \"max_uptime_in_millis\": 2021226, \"versions\": [ { \"version\": \"9.0.1.3\", \"vm_name\": \"OpenJDK 64-Bit Server VM\", \"vm_version\": \"9.0.1.3+11\", \"vm_vendor\": \"Azul Systems, Inc.\", \"bundled_jdk\": false, \"using_bundled_jdk\": null, \"count\": 1 } ], \"mem\": { \"heap_used_in_bytes\": 277003800, \"heap_max_in_bytes\": 1037959168 }, \"threads\": 66 }, \"fs\": { \"total_in_bytes\": 6000527532032, \"free_in_bytes\": 3111816585216, \"available_in_bytes\": 3111816585216 }, \"plugins\": [], \"network_types\": { \"transport_types\": { \"netty4\": 1 }, \"http_types\": { \"netty4\": 1 } }, \"discovery_types\": { \"zen\": 1 } } }" +func TestLoadESBasedElasticConfigSkipsWhenSystemClusterUnavailable(t *testing.T) { + previous := global.Lookup(coreElastic.GlobalSystemElasticsearchID) + defer global.Register(coreElastic.GlobalSystemElasticsearchID, previous) + + global.Register(coreElastic.GlobalSystemElasticsearchID, "") + + configs := loadESBasedElasticConfig() + if len(configs) != 0 { + t.Fatalf("expected no remote configs when system cluster id is unavailable, got %d", len(configs)) + } +} + +func TestElasticModuleStartSkipsSystemClusterDependentInitBeforeSetup(t *testing.T) { + previousSystemID := global.Lookup(coreElastic.GlobalSystemElasticsearchID) + defer global.Register(coreElastic.GlobalSystemElasticsearchID, previousSystemID) + + previousModuleConfig := moduleConfig + defer func() { + moduleConfig = previousModuleConfig + }() + + previousOrmInited := ormInited + defer func() { + ormInited = previousOrmInited + }() + + global.Register(coreElastic.GlobalSystemElasticsearchID, "") + + moduleConfig = getDefaultConfig() + moduleConfig.ORMConfig.Enabled = true + moduleConfig.StoreConfig.Enabled = true + moduleConfig.RemoteConfigEnabled = true + ormInited = false - d1, err := jsonparser.GetInt(util.UnsafeStringToBytes(str), "indices", "segments", "max_unsafe_auto_id_timestamp") - fmt.Println("xv:", d1, err) - if err != nil { - d, err := jsonparser.Set(util.UnsafeStringToBytes(str), []byte("-1"), "indices", "segments", "max_unsafe_auto_id_timestamp") - if err == nil { - str = util.UnsafeBytesToString(d) - } + module := &ElasticModule{} + if err := module.Start(); err != nil { + t.Fatalf("expected elastic module start to succeed before setup, got %v", err) } - d1, err = jsonparser.GetInt(util.UnsafeStringToBytes(str), "indices", "segments", "max_unsafe_auto_id_timestamp") - fmt.Println("xv:", d1, err) - //xv,err:=jsonparser.GetInt([]byte(str),"indices.segments.max_unsafe_auto_id_timestamp") - //fmt.Println("xv:",xv,err) } From 9214dc57428381ea48c88bd0b52e2cd0fd1f4e24 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 14:25:42 +0800 Subject: [PATCH 020/137] fix: console setup failed without orm handler --- core/orm/registry.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/orm/registry.go b/core/orm/registry.go index 1cf2a1f9c..ab3b4513b 100644 --- a/core/orm/registry.go +++ b/core/orm/registry.go @@ -51,6 +51,10 @@ func InitSchema() error { var handler ORM +func HasHandler() bool { + return handler != nil +} + func getHandler() ORM { if handler == nil { panic(errors.New("ORM handler is not registered")) From cd63deddad5ec146a94e8b3b459cb0579b564f31 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 15:42:17 +0800 Subject: [PATCH 021/137] fix: task and env for data and log --- core/env/env.go | 25 +++++++++++++++++++++++++ core/env/env_test.go | 30 ++++++++++++++++++++++++++++++ core/task/task.go | 24 +++++++++++++++++++++--- 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/core/env/env.go b/core/env/env.go index 84601e6b7..c680f78ce 100755 --- a/core/env/env.go +++ b/core/env/env.go @@ -418,6 +418,7 @@ func (env *Env) loadEnvFromConfigFile(filename string) error { } env.SystemConfig = &tempCfg + env.normalizeRelativePaths() //initialize node config env.findWorkingDir() @@ -481,6 +482,30 @@ func (env *Env) loadEnvFromConfigFile(filename string) error { return nil } +func resolvePathRelativeToExecutable(p string) string { + p = strings.TrimSpace(p) + if p == "" || filepath.IsAbs(p) { + return p + } + + executablePath, err := os.Executable() + if err != nil { + return p + } + return filepath.Join(filepath.Dir(executablePath), p) +} + +func (env *Env) normalizeRelativePaths() { + if env.SystemConfig == nil { + return + } + + env.SystemConfig.PathConfig.Config = resolvePathRelativeToExecutable(env.SystemConfig.PathConfig.Config) + env.SystemConfig.PathConfig.Data = resolvePathRelativeToExecutable(env.SystemConfig.PathConfig.Data) + env.SystemConfig.PathConfig.Log = resolvePathRelativeToExecutable(env.SystemConfig.PathConfig.Log) + env.SystemConfig.PathConfig.Plugin = resolvePathRelativeToExecutable(env.SystemConfig.PathConfig.Plugin) +} + func (env *Env) GetConfigFile() string { return env.configFile } diff --git a/core/env/env_test.go b/core/env/env_test.go index 23e00d752..d6b77fb17 100644 --- a/core/env/env_test.go +++ b/core/env/env_test.go @@ -24,6 +24,8 @@ package env import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -103,6 +105,34 @@ func TestParseConfigSection_ExistingKey_UnpackFails(t *testing.T) { require.Error(t, err) } +func TestResolvePathRelativeToExecutableUsesExecutableDir(t *testing.T) { + executablePath, err := os.Executable() + require.NoError(t, err) + + got := resolvePathRelativeToExecutable("data") + + assert.Equal(t, filepath.Join(filepath.Dir(executablePath), "data"), got) +} + +func TestNormalizeRelativePathsUsesExecutableDir(t *testing.T) { + executablePath, err := os.Executable() + require.NoError(t, err) + + env := EmptyEnv() + env.SystemConfig.PathConfig.Config = "config" + env.SystemConfig.PathConfig.Data = "data" + env.SystemConfig.PathConfig.Log = "log" + env.SystemConfig.PathConfig.Plugin = "plugin" + + env.normalizeRelativePaths() + + executableDir := filepath.Dir(executablePath) + assert.Equal(t, filepath.Join(executableDir, "config"), env.SystemConfig.PathConfig.Config) + assert.Equal(t, filepath.Join(executableDir, "data"), env.SystemConfig.PathConfig.Data) + assert.Equal(t, filepath.Join(executableDir, "log"), env.SystemConfig.PathConfig.Log) + assert.Equal(t, filepath.Join(executableDir, "plugin"), env.SystemConfig.PathConfig.Plugin) +} + func TestParseConfigSection_KeyExistsButPrimitive_ReturnsError(t *testing.T) { // Key exists but value is primitive (string), not an object. Child returns type error. cfg, err := config.NewConfigFrom(map[string]interface{}{ diff --git a/core/task/task.go b/core/task/task.go index 8ae5d6d0e..1b5eab23d 100644 --- a/core/task/task.go +++ b/core/task/task.go @@ -28,9 +28,11 @@ import ( log "github.com/cihub/seelog" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" + "infini.sh/framework/core/orm" "infini.sh/framework/core/task/chrono" "infini.sh/framework/core/util" "runtime" + "strings" "sync" "sync/atomic" "time" @@ -38,6 +40,22 @@ import ( var Tasks = sync.Map{} +func shouldSilenceStartupTaskError(msg string) bool { + return !orm.HasHandler() && strings.Contains(msg, "ORM handler is not registered") +} + +func logTaskRuntimeIssue(msg string, raw interface{}) { + if shouldSilenceStartupTaskError(msg) { + log.Debug(msg) + return + } + if raw != nil { + log.Error(raw, msg) + return + } + log.Error(msg) +} + type State string const ( @@ -103,7 +121,7 @@ func RegisterTransientTask(group, tag string, f func(ctx context.Context) error, case string: v = r.(string) } - log.Error(r, v) + logTaskRuntimeIssue(v, r) } } task.State = Finished @@ -118,7 +136,7 @@ func RegisterTransientTask(group, tag string, f func(ctx context.Context) error, task.State = Running err := inner(innerCtx) if err != nil { - log.Error(err) + logTaskRuntimeIssue(err.Error(), err) } t = time.Now() task.EndTime = &t @@ -194,7 +212,7 @@ func RegisterScheduleTask(task ScheduleTask) (taskID string) { case string: v = r.(string) } - log.Error(v) + logTaskRuntimeIssue(v, nil) } } task.isTaskRunning.Store(false) From 65c74c96514e2608d7f41cae41835d4188372e3a Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 16:42:03 +0800 Subject: [PATCH 022/137] fix: service install and start user corrent data and log path --- app.go | 18 ++++++++++++++---- app_test.go | 20 ++++++++++++++++++++ core/env/env.go | 6 +++++- core/env/env_test.go | 21 +++++++++++++++++++++ 4 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 app_test.go diff --git a/app.go b/app.go index f810811ae..af1ef24e5 100755 --- a/app.go +++ b/app.go @@ -38,6 +38,7 @@ import ( "infini.sh/framework/modules/configs/client" "os" "os/signal" + "path/filepath" "runtime" "runtime/debug" "sync" @@ -84,6 +85,18 @@ type App struct { svcFlag string } +func getServiceWorkingDirectory() string { + executablePath, err := os.Executable() + if err == nil { + return filepath.Dir(executablePath) + } + workdir, err := os.Getwd() + if err != nil { + panic(err) + } + return workdir +} + const ( env_SILENT_GREETINGS = "SILENT_GREETINGS" env_SERVICE_NAME = "SERVICE_NAME" @@ -574,10 +587,7 @@ func (app *App) Run() { svcOptions["SuccessExitStatus"] = "1 2 8 SIGKILL" svcOptions["LimitNOFILE"] = 1024000 - workdir, err := os.Getwd() - if err != nil { - panic(err) - } + workdir := getServiceWorkingDirectory() serviceName := app.environment.GetAppLowercaseName() if v, ok := os.LookupEnv(env_SERVICE_NAME); ok { diff --git a/app_test.go b/app_test.go new file mode 100644 index 000000000..702a85f48 --- /dev/null +++ b/app_test.go @@ -0,0 +1,20 @@ +package framework + +import ( + "os" + "path/filepath" + "testing" +) + +func TestGetServiceWorkingDirectoryUsesExecutableDir(t *testing.T) { + executablePath, err := os.Executable() + if err != nil { + t.Fatalf("failed to get executable path: %v", err) + } + + got := getServiceWorkingDirectory() + want := filepath.Dir(executablePath) + if got != want { + t.Fatalf("expected service working directory %q, got %q", want, got) + } +} diff --git a/core/env/env.go b/core/env/env.go index c680f78ce..1744e37f3 100755 --- a/core/env/env.go +++ b/core/env/env.go @@ -255,7 +255,11 @@ func (env *Env) InitPaths(cfgPath string) error { if cfgObj, err = config.LoadFile(cfgPath); err != nil { return fmt.Errorf("error loading confiuration file: %v, %w", cfgPath, err) } - return cfgObj.Unpack(&env.SystemConfig) + if err := cfgObj.Unpack(&env.SystemConfig); err != nil { + return err + } + env.normalizeRelativePaths() + return nil } else { if !env.IgnoreOnConfigMissing { return errors.Errorf("config file %v not found", cfgPath) diff --git a/core/env/env_test.go b/core/env/env_test.go index d6b77fb17..b87292741 100644 --- a/core/env/env_test.go +++ b/core/env/env_test.go @@ -133,6 +133,27 @@ func TestNormalizeRelativePathsUsesExecutableDir(t *testing.T) { assert.Equal(t, filepath.Join(executableDir, "plugin"), env.SystemConfig.PathConfig.Plugin) } +func TestInitPathsNormalizesRelativePathsFromConfig(t *testing.T) { + executablePath, err := os.Executable() + require.NoError(t, err) + + cfgFile, err := os.CreateTemp("", "env-paths-*.yml") + require.NoError(t, err) + defer os.Remove(cfgFile.Name()) + + _, err = cfgFile.WriteString("path.data: data\npath.log: log\npath.configs: config\n") + require.NoError(t, err) + require.NoError(t, cfgFile.Close()) + + env := EmptyEnv() + require.NoError(t, env.InitPaths(cfgFile.Name())) + + executableDir := filepath.Dir(executablePath) + assert.Equal(t, filepath.Join(executableDir, "data"), env.SystemConfig.PathConfig.Data) + assert.Equal(t, filepath.Join(executableDir, "log"), env.SystemConfig.PathConfig.Log) + assert.Equal(t, filepath.Join(executableDir, "config"), env.SystemConfig.PathConfig.Config) +} + func TestParseConfigSection_KeyExistsButPrimitive_ReturnsError(t *testing.T) { // Key exists but value is primitive (string), not an object. Child returns type error. cfg, err := config.NewConfigFrom(map[string]interface{}{ From 102a66b4ab69ad503f071a914d7162216b176761 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 18:10:21 +0800 Subject: [PATCH 023/137] improve: patition hash with missing values --- core/elastic/partition.go | 41 ++++++++++++++++--------------- core/elastic/partition_test.go | 45 ++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 20 deletions(-) diff --git a/core/elastic/partition.go b/core/elastic/partition.go index d4c0127d6..8f6565dea 100644 --- a/core/elastic/partition.go +++ b/core/elastic/partition.go @@ -754,7 +754,7 @@ func buildHashPartitionFilter(partitionID, partitionCount int, fieldName string, "script": util.MapStr{ "script": util.MapStr{ "lang": "painless", - "source": fmt.Sprintf("doc[%s].size()!=0 && (((doc[%s].value.hashCode() %% params.partition_count) + params.partition_count) %% params.partition_count) == params.partition_id", fieldLiteral, fieldLiteral), + "source": fmt.Sprintf("doc[%s].size()!=0 && doc[%s].value != '' && (((doc[%s].value.hashCode() %% params.partition_count) + params.partition_count) %% params.partition_count) == params.partition_id", fieldLiteral, fieldLiteral, fieldLiteral), "params": util.MapStr{ "partition_count": partitionCount, "partition_id": partitionID, @@ -820,34 +820,35 @@ func getJSONPathString(data []byte, path ...string) (string, bool) { func buildMissingFieldCondition(fieldName string) util.MapStr { return util.MapStr{ "bool": util.MapStr{ - "must_not": []interface{}{ + "should": []interface{}{ util.MapStr{ - "exists": util.MapStr{ - "field": fieldName, + "bool": util.MapStr{ + "must_not": []interface{}{ + util.MapStr{ + "exists": util.MapStr{ + "field": fieldName, + }, + }, + }, + }, + }, + util.MapStr{ + "term": util.MapStr{ + fieldName: util.MapStr{ + "value": "", + }, }, }, }, + "minimum_should_match": 1, }, } } func buildMissingFieldFilter(fieldName string, filter interface{}) util.MapStr { - boolFilter := util.MapStr{ - "must": []interface{}{}, - "must_not": []interface{}{ - util.MapStr{ - "exists": util.MapStr{ - "field": fieldName, - }, - }, - }, - } - if filter != nil { - boolFilter["must"] = append(boolFilter["must"].([]interface{}), filter) - } - return util.MapStr{ - "bool": boolFilter, - } + return buildMustPartitionFilter([]interface{}{ + buildMissingFieldCondition(fieldName), + }, filter) } func buildMustPartitionFilter(mustClauses []interface{}, filter interface{}) util.MapStr { diff --git a/core/elastic/partition_test.go b/core/elastic/partition_test.go index b07219511..f6cf788e5 100644 --- a/core/elastic/partition_test.go +++ b/core/elastic/partition_test.go @@ -97,6 +97,9 @@ func TestBuildHashPartitionFilter(t *testing.T) { if !strings.Contains(source, "doc['pmid.keyword']") { t.Fatalf("unexpected script source: %s", source) } + if !strings.Contains(source, "value != ''") { + t.Fatalf("expected empty strings to be excluded from hash partition, got %s", source) + } if strings.Contains(source, "Math.floorMod") { t.Fatalf("unexpected script source: %s", source) } @@ -109,6 +112,48 @@ func TestBuildHashPartitionFilter(t *testing.T) { } } +func TestBuildMissingFieldConditionIncludesEmptyString(t *testing.T) { + filter := buildMissingFieldCondition("pmid.keyword") + boolFilter, ok := filter["bool"].(util.MapStr) + if !ok { + t.Fatalf("expected bool filter, got %v", filter) + } + if got := boolFilter["minimum_should_match"]; got != 1 { + t.Fatalf("unexpected minimum_should_match: %v", got) + } + should, ok := boolFilter["should"].([]interface{}) + if !ok || len(should) != 2 { + t.Fatalf("expected two should clauses, got %v", boolFilter["should"]) + } + termFilter := should[1].(util.MapStr)["term"].(util.MapStr)["pmid.keyword"].(util.MapStr) + if got := termFilter["value"]; got != "" { + t.Fatalf("unexpected empty-string term filter: %v", termFilter) + } +} + +func TestBuildMissingFieldFilterPreservesOuterFilter(t *testing.T) { + filter := buildMissingFieldFilter("pmid.keyword", util.MapStr{ + "term": util.MapStr{ + "env": util.MapStr{"value": "prod"}, + }, + }) + boolFilter, ok := filter["bool"].(util.MapStr) + if !ok { + t.Fatalf("expected bool filter, got %v", filter) + } + must, ok := boolFilter["must"].([]interface{}) + if !ok || len(must) != 2 { + t.Fatalf("expected two must clauses, got %v", boolFilter["must"]) + } + innerBool, ok := must[0].(util.MapStr)["bool"].(util.MapStr) + if !ok { + t.Fatalf("expected wrapped missing bool filter, got %v", must[0]) + } + if got := innerBool["minimum_should_match"]; got != 1 { + t.Fatalf("unexpected minimum_should_match: %v", got) + } +} + func TestBuildPainlessStringLiteralEscapesSingleQuote(t *testing.T) { got := buildPainlessStringLiteral("foo'bar") if got != `'foo\'bar'` { From 88faccffd44664bd20a680c99587b00be8b93fe8 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 20:02:12 +0800 Subject: [PATCH 024/137] improve: pipeline with delete task --- modules/pipeline/pipeline.go | 5 ++++ modules/pipeline/pipeline_test.go | 50 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 modules/pipeline/pipeline_test.go diff --git a/modules/pipeline/pipeline.go b/modules/pipeline/pipeline.go index 25772b200..700ebaa64 100755 --- a/modules/pipeline/pipeline.go +++ b/modules/pipeline/pipeline.go @@ -150,6 +150,11 @@ func (module *PipeModule) stopTask(taskID string) (exists bool) { // deleteTask will clean all in-memory states and release the pipeline context func (module *PipeModule) deleteTask(taskID string) { + if ctx, ok := module.contexts.Load(taskID); ok { + if v1, ok := ctx.(*pipeline.Context); ok && !v1.IsLoopReleased() { + module.stopAndWaitForRelease([]string{taskID}, time.Minute) + } + } module.pipelines.Delete(taskID) module.configs.Delete(taskID) module.releaseContext(taskID) diff --git a/modules/pipeline/pipeline_test.go b/modules/pipeline/pipeline_test.go new file mode 100644 index 000000000..a34b123ea --- /dev/null +++ b/modules/pipeline/pipeline_test.go @@ -0,0 +1,50 @@ +package pipeline + +import ( + "testing" + "time" + + corepipeline "infini.sh/framework/core/pipeline" +) + +func TestDeleteTaskWaitsForLoopRelease(t *testing.T) { + module := &PipeModule{} + ctx := corepipeline.AcquireContext(corepipeline.PipelineConfigV2{}) + + module.contexts.Store("task-1", ctx) + module.configs.Store("task-1", corepipeline.PipelineConfigV2{Name: "task-1"}) + module.pipelines.Store("task-1", struct{}{}) + + released := make(chan struct{}) + go func() { + for !ctx.IsCanceled() { + time.Sleep(time.Millisecond) + } + time.Sleep(50 * time.Millisecond) + ctx.SetLoopReleased() + close(released) + }() + + start := time.Now() + module.deleteTask("task-1") + elapsed := time.Since(start) + + select { + case <-released: + default: + t.Fatal("expected deleteTask to wait for loop release") + } + + if elapsed < 50*time.Millisecond { + t.Fatalf("expected deleteTask to wait for loop release, returned after %v", elapsed) + } + if _, ok := module.contexts.Load("task-1"); ok { + t.Fatal("expected context to be deleted") + } + if _, ok := module.configs.Load("task-1"); ok { + t.Fatal("expected config to be deleted") + } + if _, ok := module.pipelines.Load("task-1"); ok { + t.Fatal("expected pipeline to be deleted") + } +} From efa8d4dbc891381a6056da80b055c9cd47707470 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 20:46:18 +0800 Subject: [PATCH 025/137] improve: bulk index with docs --- plugins/elastic/bulk_indexing/bulk_indexing.go | 14 +++++++++++--- .../elastic/bulk_indexing/bulk_indexing_test.go | 8 +++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index 637e6bd57..9c96a8b3e 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -319,7 +319,12 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { if processor.config.DetectIntervalInMs > 0 { time.Sleep(time.Millisecond * time.Duration(processor.config.DetectIntervalInMs)) } - if shouldQuitActiveQueueDetection(lastDispatch, time.Duration(processor.config.IdleTimeoutInSecond)*time.Second, util.MapLength(&processor.inFlightQueueConfigs)) { + if shouldQuitActiveQueueDetection( + lastDispatch, + time.Duration(processor.config.IdleTimeoutInSecond)*time.Second, + time.Duration(processor.config.DetectIntervalInMs)*time.Millisecond, + util.MapLength(&processor.inFlightQueueConfigs), + ) { return } } @@ -343,11 +348,14 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { return nil } -func shouldQuitActiveQueueDetection(lastDispatch time.Time, idleDuration time.Duration, inflight int) bool { +func shouldQuitActiveQueueDetection(lastDispatch time.Time, idleDuration time.Duration, detectInterval time.Duration, inflight int) bool { if idleDuration <= 0 { return false } - return inflight == 0 && time.Since(lastDispatch) > idleDuration + if detectInterval < 0 { + detectInterval = 0 + } + return inflight == 0 && time.Since(lastDispatch) >= idleDuration+detectInterval } const queueHandleSingleton = "queue_handler_singleton" diff --git a/plugins/elastic/bulk_indexing/bulk_indexing_test.go b/plugins/elastic/bulk_indexing/bulk_indexing_test.go index f3cfa468a..da977cab2 100644 --- a/plugins/elastic/bulk_indexing/bulk_indexing_test.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing_test.go @@ -157,7 +157,9 @@ func TestIsIgnorableAcquireConsumerError(t *testing.T) { } func TestShouldQuitActiveQueueDetection(t *testing.T) { - assert.False(t, shouldQuitActiveQueueDetection(time.Now(), 5*time.Second, 0)) - assert.False(t, shouldQuitActiveQueueDetection(time.Now().Add(-10*time.Second), 5*time.Second, 1)) - assert.True(t, shouldQuitActiveQueueDetection(time.Now().Add(-10*time.Second), 5*time.Second, 0)) + assert.False(t, shouldQuitActiveQueueDetection(time.Now(), 5*time.Second, 5*time.Second, 0)) + assert.False(t, shouldQuitActiveQueueDetection(time.Now().Add(-10*time.Second), 5*time.Second, 5*time.Second, 1)) + assert.False(t, shouldQuitActiveQueueDetection(time.Now().Add(-9*time.Second), 5*time.Second, 5*time.Second, 0)) + assert.True(t, shouldQuitActiveQueueDetection(time.Now().Add(-10*time.Second), 5*time.Second, 5*time.Second, 0)) + assert.True(t, shouldQuitActiveQueueDetection(time.Now().Add(-5*time.Second), 5*time.Second, 0, 0)) } From 9cbed71a6809eb1c381a74bb24d9c998c125bcdc Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 21:10:39 +0800 Subject: [PATCH 026/137] fix: pecentail for date --- core/elastic/partition.go | 21 +++++++++++++++++++++ core/elastic/partition_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/core/elastic/partition.go b/core/elastic/partition.go index 8f6565dea..0a9c49abd 100644 --- a/core/elastic/partition.go +++ b/core/elastic/partition.go @@ -652,6 +652,8 @@ func buildBoundedPartitionFilter(min, max float64, fieldName, fieldType string, "lte": max, } if fieldType == PartitionByDate { + rv["gte"] = normalizeDateRangeBoundary(min, true, true) + rv["lte"] = normalizeDateRangeBoundary(max, false, true) rv["format"] = "epoch_millis" } must := []interface{}{ @@ -680,6 +682,12 @@ func buildOpenPartitionFilter(lower, upper *float64, fieldName, fieldType string rv["lte"] = *upper } if fieldType == PartitionByDate { + if lower != nil { + rv["gt"] = normalizeDateRangeBoundary(*lower, true, false) + } + if upper != nil { + rv["lte"] = normalizeDateRangeBoundary(*upper, false, true) + } rv["format"] = "epoch_millis" } var condition interface{} @@ -708,6 +716,19 @@ func buildOpenPartitionFilter(lower, upper *float64, fieldName, fieldType string } +func normalizeDateRangeBoundary(value float64, lower, inclusive bool) int64 { + switch { + case lower && inclusive: + return int64(math.Ceil(value)) + case lower && !inclusive: + return int64(math.Floor(value)) + case !lower && inclusive: + return int64(math.Floor(value)) + default: + return int64(math.Ceil(value)) + } +} + func buildExactTermPartitionFilter(value, fieldName string, filter interface{}) util.MapStr { return buildMustPartitionFilter([]interface{}{ util.MapStr{ diff --git a/core/elastic/partition_test.go b/core/elastic/partition_test.go index f6cf788e5..bee6b587c 100644 --- a/core/elastic/partition_test.go +++ b/core/elastic/partition_test.go @@ -63,6 +63,33 @@ func TestBuildOpenPartitionFilterPreservesDateFormat(t *testing.T) { if got := rangeFilter["format"]; got != "epoch_millis" { t.Fatalf("unexpected date format: %v", got) } + if got := rangeFilter["lte"]; got != int64(1000) { + t.Fatalf("unexpected upper bound: %v", got) + } +} + +func TestBuildOpenPartitionFilterRoundsDatePercentileBoundaries(t *testing.T) { + lower := 1779109187904.8455 + upper := 1779109187999.999 + filter := buildOpenPartitionFilter(&lower, &upper, "created_at", PartitionByDate, nil) + rangeFilter := getMustClause(t, filter)["range"].(util.MapStr)["created_at"].(util.MapStr) + if got := rangeFilter["gt"]; got != int64(1779109187904) { + t.Fatalf("unexpected lower bound: %v", got) + } + if got := rangeFilter["lte"]; got != int64(1779109187999) { + t.Fatalf("unexpected upper bound: %v", got) + } +} + +func TestBuildBoundedPartitionFilterRoundsDateBoundaries(t *testing.T) { + filter := buildBoundedPartitionFilter(1779109187904.1, 1779109187999.9, "created_at", PartitionByDate, nil) + rangeFilter := getMustClause(t, filter)["range"].(util.MapStr)["created_at"].(util.MapStr) + if got := rangeFilter["gte"]; got != int64(1779109187905) { + t.Fatalf("unexpected lower bound: %v", got) + } + if got := rangeFilter["lte"]; got != int64(1779109187999) { + t.Fatalf("unexpected upper bound: %v", got) + } } func TestBuildExactTermPartitionFilter(t *testing.T) { From 94951c342e63b149a946e6887c76d65953382a2a Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 18 May 2026 21:31:56 +0800 Subject: [PATCH 027/137] improve: partition with terms --- core/elastic/partition.go | 83 +++++++++++++++++++++++++++++++++- core/elastic/partition_test.go | 58 ++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/core/elastic/partition.go b/core/elastic/partition.go index 0a9c49abd..d18855642 100644 --- a/core/elastic/partition.go +++ b/core/elastic/partition.go @@ -401,7 +401,7 @@ func getPartitionsByHash(client API, indexName, fieldName string, partitionCount }) } - counts, err := getPartitionDocCounts(client, indexName, partitions) + counts, err := getHashPartitionDocCounts(client, indexName, fieldName, partitionCount, filter) if err != nil { return nil, err } @@ -417,6 +417,87 @@ func getPartitionsByHash(client API, indexName, fieldName string, partitionCount return filtered, nil } +func getHashPartitionDocCounts(client API, indexName, fieldName string, partitionCount int, filter interface{}) ([]int64, error) { + queryDsl := buildHashPartitionAggQuery(fieldName, partitionCount, filter) + res, err := searchPartitionWithRawQueryDSL(client, indexName, queryDsl) + if err != nil { + return nil, err + } + return extractHashPartitionDocCounts(res, partitionCount), nil +} + +func buildHashPartitionAggQuery(fieldName string, partitionCount int, filter interface{}) util.MapStr { + fieldLiteral := buildPainlessStringLiteral(fieldName) + queryDsl := util.MapStr{ + "size": 0, + "aggs": util.MapStr{ + "partitions": util.MapStr{ + "terms": util.MapStr{ + "size": partitionCount, + "value_type": "long", + "script": util.MapStr{ + "lang": "painless", + "source": fmt.Sprintf("if (doc[%s].size()==0 || doc[%s].value == '') return null; return (((doc[%s].value.hashCode() %% params.partition_count) + params.partition_count) %% params.partition_count);", fieldLiteral, fieldLiteral, fieldLiteral), + "params": util.MapStr{ + "partition_count": partitionCount, + }, + }, + }, + }, + }, + } + if filter != nil { + queryDsl["query"] = filter + } + return queryDsl +} + +func extractHashPartitionDocCounts(res *SearchResponse, partitionCount int) []int64 { + counts := make([]int64, partitionCount) + if res == nil { + return counts + } + partitionsAgg, ok := res.Aggregations["partitions"] + if !ok { + return counts + } + for _, bucket := range partitionsAgg.Buckets { + bucketKey, ok := extractHashPartitionBucketKey(bucket["key"]) + if !ok || bucketKey < 0 || bucketKey >= partitionCount { + continue + } + counts[bucketKey] = util.GetInt64Value(bucket["doc_count"]) + } + return counts +} + +func extractHashPartitionBucketKey(key interface{}) (int, bool) { + switch v := key.(type) { + case int: + return v, true + case int64: + return int(v), true + case int32: + return int(v), true + case uint: + return int(v), true + case uint64: + return int(v), true + case float64: + return int(v), true + case float32: + return int(v), true + case string: + parsed, err := strconv.Atoi(v) + if err != nil { + return 0, false + } + return parsed, true + default: + return 0, false + } +} + func getQuantileBoundaries(client API, indexName, fieldName string, partitionCount int, min, max float64, filter interface{}) ([]float64, error) { percents := buildQuantilePercents(partitionCount) if len(percents) == 0 { diff --git a/core/elastic/partition_test.go b/core/elastic/partition_test.go index bee6b587c..1a7795a6d 100644 --- a/core/elastic/partition_test.go +++ b/core/elastic/partition_test.go @@ -139,6 +139,64 @@ func TestBuildHashPartitionFilter(t *testing.T) { } } +func TestBuildHashPartitionAggQueryAppliesOuterFilter(t *testing.T) { + query := buildHashPartitionAggQuery("pmid.keyword", 8, util.MapStr{ + "term": util.MapStr{ + "env": util.MapStr{"value": "prod"}, + }, + }) + + if !reflect.DeepEqual(query["query"], util.MapStr{ + "term": util.MapStr{ + "env": util.MapStr{"value": "prod"}, + }, + }) { + t.Fatalf("expected outer filter to be applied at top-level query, got %v", query["query"]) + } + + termsAgg := query["aggs"].(util.MapStr)["partitions"].(util.MapStr)["terms"].(util.MapStr) + if got := termsAgg["size"]; got != 8 { + t.Fatalf("unexpected partition size: %v", got) + } + if got := termsAgg["value_type"]; got != "long" { + t.Fatalf("unexpected value_type: %v", got) + } + script := termsAgg["script"].(util.MapStr) + source, ok := script["source"].(string) + if !ok { + t.Fatalf("unexpected script source type: %T", script["source"]) + } + if !strings.Contains(source, "return null") { + t.Fatalf("expected missing values to be skipped in hash aggregation, got %s", source) + } + if !strings.Contains(source, "value == ''") { + t.Fatalf("expected empty strings to be excluded in hash aggregation, got %s", source) + } + params := script["params"].(util.MapStr) + if got := params["partition_count"]; got != 8 { + t.Fatalf("unexpected partition_count: %v", got) + } +} + +func TestExtractHashPartitionDocCountsMapsByBucketKey(t *testing.T) { + counts := extractHashPartitionDocCounts(&SearchResponse{ + Aggregations: map[string]AggregationResponse{ + "partitions": { + Buckets: []BucketBase{ + {"key": float64(5), "doc_count": float64(12)}, + {"key": "1", "doc_count": float64(7)}, + {"key": float64(99), "doc_count": float64(3)}, + }, + }, + }, + }, 8) + + expected := []int64{0, 7, 0, 0, 0, 12, 0, 0} + if !reflect.DeepEqual(counts, expected) { + t.Fatalf("unexpected hash counts: got %v want %v", counts, expected) + } +} + func TestBuildMissingFieldConditionIncludesEmptyString(t *testing.T) { filter := buildMissingFieldCondition("pmid.keyword") boolFilter, ok := filter["bool"].(util.MapStr) From eaaac5ed80ad269104ee283f77e4340a9456bafb Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 19 May 2026 05:37:35 +0800 Subject: [PATCH 028/137] fix: bulk index with corrent offset --- plugins/elastic/bulk_indexing/bulk_indexing.go | 18 +++++++++--------- .../bulk_indexing/bulk_indexing_test.go | 12 ++++++++++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index 9c96a8b3e..308144ab7 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -1012,6 +1012,10 @@ READ_DOCS: mainBuf.WriteByteBuffer(pop.Data) } + // Keep the in-memory offset aligned with the data already buffered. + // If the current message triggers an immediate flush, its NextOffset must be committed too. + offset = advanceBufferedOffset(pop.NextOffset) + if global.Env().IsDebug { log.Tracef("slice worker, worker:[%v], message count: %v, size: %v", workerID, mainBuf.GetMessageCount(), util.ByteSize(uint64(mainBuf.GetMessageSize()))) } @@ -1071,21 +1075,12 @@ READ_DOCS: } else { // skip unchanged offset silently to avoid noisy debug logs } - // fix: this code is moved to loop outside (line 970) to avoid updating offset in the middle of bulk submission - // offset = &pop.NextOffset } } else { log.Errorf("should not submit this bulk request, worker[%v], queue:[%v], slice:[%v], offset:[%v]->[%v],%v, msg:%v", workerID, qConfig.ID, sliceID, committedOffset, offset, err, msgCount) } } - - // fix: update offset after each message is processed, to ensure progress sync with actual processing - // so even if it crashes before submission, it will not repeat processing messages written to the buffer after restart - offset = &pop.NextOffset } - - // fix: remove this code to avoid overwriting the updated offset in the loop - // offset = &ctx1.NextOffset } if time.Since(lastCommit) > idleDuration && mainBuf.GetMessageSize() > 0 { @@ -1274,6 +1269,11 @@ func appendStrArr(arr []string, size int, elems []string) []string { return append(arr, elems...) } +func advanceBufferedOffset(nextOffset queue.Offset) *queue.Offset { + next := nextOffset + return &next +} + func (processor *BulkIndexingProcessor) getElasticsearchMetadata(qConfig *queue.QueueConfig) (string, *elastic.ElasticsearchMetadata) { elasticsearch, ok := qConfig.Labels["elasticsearch"] diff --git a/plugins/elastic/bulk_indexing/bulk_indexing_test.go b/plugins/elastic/bulk_indexing/bulk_indexing_test.go index da977cab2..f462d8f34 100644 --- a/plugins/elastic/bulk_indexing/bulk_indexing_test.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing_test.go @@ -31,6 +31,7 @@ import ( stdErrors "errors" "github.com/OneOfOne/xxhash" "github.com/stretchr/testify/assert" + "infini.sh/framework/core/queue" "sync" "testing" "time" @@ -163,3 +164,14 @@ func TestShouldQuitActiveQueueDetection(t *testing.T) { assert.True(t, shouldQuitActiveQueueDetection(time.Now().Add(-10*time.Second), 5*time.Second, 5*time.Second, 0)) assert.True(t, shouldQuitActiveQueueDetection(time.Now().Add(-5*time.Second), 5*time.Second, 0, 0)) } + +func TestAdvanceBufferedOffsetUsesCurrentMessageNextOffset(t *testing.T) { + previousCommitted := queue.NewOffsetWithVersion(0, 100, 1) + currentNext := queue.NewOffsetWithVersion(0, 200, 1) + + offset := advanceBufferedOffset(currentNext) + + assert.NotNil(t, offset) + assert.True(t, offset.Equals(currentNext)) + assert.False(t, offset.Equals(previousCommitted)) +} From 88b2394755809c5163a697efe5b48dd991a576d8 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 19 May 2026 10:14:35 +0800 Subject: [PATCH 029/137] improve: format with only go files --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 431aaf49c..2a1243ab9 100755 --- a/Makefile +++ b/Makefile @@ -262,7 +262,10 @@ cross-build-all-platform: clean config build-bsd build-linux build-darwin build- format: @echo "formatting code" - GOPATH=$(NEWGOPATH) $(GO) fmt $$(GOPATH=$(NEWGOPATH) $(GO) list ./...) + find . -type f -name '*.go' \ + -not -path './vendor/*' \ + -not -path './.git/*' \ + -print0 | xargs -0 gofmt -w test: config $(GOTEST) -v $(GOFLAGS) -timeout 30m ./... @@ -399,4 +402,4 @@ package-linux-arm-platform: package-windows-platform: @echo "Packaging Windows" cd $(OUTPUT_DIR) && zip -r $(OUTPUT_DIR)/windows-amd64.zip $(APP_NAME)-windows-amd64.exe $(APP_CONFIG) - cd $(OUTPUT_DIR) && zip -r $(OUTPUT_DIR)/windows-386.zip $(APP_NAME)-windows-386.exe $(APP_CONFIG) \ No newline at end of file + cd $(OUTPUT_DIR) && zip -r $(OUTPUT_DIR)/windows-386.zip $(APP_NAME)-windows-386.exe $(APP_CONFIG) From bea7e1d12cb50d44e85a589fba0b86babd2f2bd2 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 19 May 2026 16:32:31 +0800 Subject: [PATCH 030/137] improve: add debug log for check cluster is not available --- core/elastic/actions.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/core/elastic/actions.go b/core/elastic/actions.go index dcc11b03e..f96359f27 100644 --- a/core/elastic/actions.go +++ b/core/elastic/actions.go @@ -118,10 +118,33 @@ func (node *NodeAvailable) IsDead() bool { } func (meta *ElasticsearchMetadata) IsAvailable() bool { - if meta.Config == nil || !meta.Config.Enabled { + if meta.Config == nil { + if rate.GetRateLimiter("cluster_available_check", "nil_config", 1, 1, 30*time.Second).Allow() { + log.Debug("elasticsearch metadata is unavailable: config is nil") + } + return false + } + if !meta.Config.Enabled { + clusterID := meta.Config.ID + if clusterID == "" { + clusterID = meta.Config.Name + } + if rate.GetRateLimiter("cluster_available_check", clusterID, 1, 1, 30*time.Second).Allow() { + log.Debugf("elasticsearch [%v] is unavailable: config disabled", meta.Config.Name) + } + return false + } + if !meta.clusterAvailable { + clusterID := meta.Config.ID + if clusterID == "" { + clusterID = meta.Config.Name + } + if rate.GetRateLimiter("cluster_available_check", clusterID, 1, 1, 30*time.Second).Allow() { + log.Debugf("elasticsearch [%v] is unavailable: clusterAvailable=false", meta.Config.Name) + } return false } - return meta.clusterAvailable + return true } func (meta *ElasticsearchMetadata) Init(health bool) { From 6c7e2b05b1272c4fd773fb107f3aa54751db58bb Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 19 May 2026 18:25:00 +0800 Subject: [PATCH 031/137] fix: when host can't access the metrics not use endpoint for collect --- modules/elastic/common/config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/elastic/common/config.go b/modules/elastic/common/config.go index eeadb3dd8..64f610b5d 100644 --- a/modules/elastic/common/config.go +++ b/modules/elastic/common/config.go @@ -87,7 +87,9 @@ func InitClientWithConfig(esConfig elastic.ElasticsearchConfig) (client elastic. ver string ) if esConfig.Version == "" || esConfig.Version == "auto" { - verInfo, err := adapter.ClusterVersion(elastic.GetOrInitMetadata(&esConfig)) + probeMeta := &elastic.ElasticsearchMetadata{Config: &esConfig} + probeMeta.Init(true) + verInfo, err := adapter.ClusterVersion(probeMeta) if err != nil { return nil, err } From b8b3ad2c00350f55bc43367a99a415204db3bd51 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 20 May 2026 09:56:38 +0800 Subject: [PATCH 032/137] improve: add log for migration debug --- .../elastic/bulk_indexing/bulk_indexing.go | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index 308144ab7..02544a5fe 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -235,7 +235,17 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { log.Error("error in bulk indexing processor,", v) } } - log.Debug("exit bulk indexing processor") + if processor.bulkStats != nil { + log.Debugf( + "exit bulk indexing processor, success=%d, invalid=%d, failure=%d, error_msgs=%d", + processor.bulkStats.Summary.Success.Count, + processor.bulkStats.Summary.Invalid.Count, + processor.bulkStats.Summary.Failure.Count, + len(processor.bulkStats.ErrorMsgs), + ) + } else { + log.Debug("exit bulk indexing processor") + } }() //handle updates @@ -325,6 +335,15 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { time.Duration(processor.config.DetectIntervalInMs)*time.Millisecond, util.MapLength(&processor.inFlightQueueConfigs), ) { + if processor.bulkStats != nil { + log.Debugf( + "active queue detector idle exit, success=%d, invalid=%d, failure=%d, inflight=%d", + processor.bulkStats.Summary.Success.Count, + processor.bulkStats.Summary.Invalid.Count, + processor.bulkStats.Summary.Failure.Count, + util.MapLength(&processor.inFlightQueueConfigs), + ) + } return } } From 4e40610a945734591529e6adfa1f0545bccf49ec Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 20 May 2026 11:43:47 +0800 Subject: [PATCH 033/137] improve: init delay for task --- core/task/chrono/task.go | 28 +++- core/task/chrono/task_test.go | 10 ++ core/task/task.go | 38 +++-- core/task/task_test.go | 27 ++++ modules/metrics/elastic/elasticsearch.go | 147 ++++++++++++------ modules/metrics/elastic/elasticsearch_test.go | 31 ++++ 6 files changed, 215 insertions(+), 66 deletions(-) create mode 100644 core/task/task_test.go diff --git a/core/task/chrono/task.go b/core/task/chrono/task.go index d0b6aa8f8..b626b241d 100755 --- a/core/task/chrono/task.go +++ b/core/task/chrono/task.go @@ -35,9 +35,10 @@ import ( type Task func(ctx context.Context) type SchedulerTask struct { - task Task - startTime time.Time - location *time.Location + task Task + startTime time.Time + initialDelay time.Duration + location *time.Location } func CreateSchedulerTask(task Task, options ...Option) (*SchedulerTask, error) { @@ -46,9 +47,10 @@ func CreateSchedulerTask(task Task, options ...Option) (*SchedulerTask, error) { } runnableTask := &SchedulerTask{ - task: task, - startTime: time.Time{}, - location: time.Local, + task: task, + startTime: time.Time{}, + initialDelay: 0, + location: time.Local, } for _, option := range options { @@ -63,6 +65,10 @@ func CreateSchedulerTask(task Task, options ...Option) (*SchedulerTask, error) { } func (task *SchedulerTask) GetInitialDelay() time.Duration { + if task.initialDelay > 0 { + return task.initialDelay + } + if task.startTime.IsZero() { return 0 } @@ -87,6 +93,16 @@ func WithStartTime(year int, month time.Month, day, hour, min, sec int) Option { } } +func WithInitialDelay(delay time.Duration) Option { + return func(task *SchedulerTask) error { + if delay < 0 { + delay = 0 + } + task.initialDelay = delay + return nil + } +} + func WithLocation(location string) Option { return func(task *SchedulerTask) error { loadedLocation, err := time.LoadLocation(location) diff --git a/core/task/chrono/task_test.go b/core/task/chrono/task_test.go index b154c7da5..26adbadf3 100755 --- a/core/task/chrono/task_test.go +++ b/core/task/chrono/task_test.go @@ -53,6 +53,16 @@ func TestNewSchedulerTask_WithInvalidLocation(t *testing.T) { assert.Error(t, err) } +func TestNewSchedulerTask_WithInitialDelay(t *testing.T) { + task, err := CreateSchedulerTask(func(ctx context.Context) { + }, WithInitialDelay(200*time.Millisecond)) + assert.Nil(t, err) + + delay := task.GetInitialDelay() + assert.Greater(t, delay, 0*time.Millisecond) + assert.LessOrEqual(t, delay, 200*time.Millisecond) +} + func TestNewScheduledRunnableTask(t *testing.T) { task, _ := CreateScheduledRunnableTask(0, func(ctx context.Context) { diff --git a/core/task/task.go b/core/task/task.go index 1b5eab23d..a3c207e27 100644 --- a/core/task/task.go +++ b/core/task/task.go @@ -146,15 +146,16 @@ func RegisterTransientTask(group, tag string, f func(ctx context.Context) error, } type ScheduleTask struct { - ID string `config:"id" json:"id,omitempty"` - Group string `config:"group" json:"group,omitempty"` - Description string `config:"description" json:"description,omitempty"` - Type string `config:"type" json:"type,omitempty"` - Interval string `config:"interval" json:"interval,omitempty"` - Crontab string `config:"crontab" json:"crontab,omitempty"` - CreateTime time.Time `config:"create_time" json:"create_time,omitempty"` - StartTime *time.Time `config:"start_time" json:"start_time,omitempty"` - EndTime *time.Time `config:"end_time" json:"end_time,omitempty"` + ID string `config:"id" json:"id,omitempty"` + Group string `config:"group" json:"group,omitempty"` + Description string `config:"description" json:"description,omitempty"` + Type string `config:"type" json:"type,omitempty"` + Interval string `config:"interval" json:"interval,omitempty"` + InitialDelay string `config:"initial_delay" json:"initial_delay,omitempty"` + Crontab string `config:"crontab" json:"crontab,omitempty"` + CreateTime time.Time `config:"create_time" json:"create_time,omitempty"` + StartTime *time.Time `config:"start_time" json:"start_time,omitempty"` + EndTime *time.Time `config:"end_time" json:"end_time,omitempty"` // Ensures the task runs as a singleton, preventing duplicate executions when previous attempt is not finished. Singleton bool `config:"singleton" json:"singleton,omitempty"` @@ -250,6 +251,23 @@ var taskScheduler = chrono.NewDefaultTaskScheduler() var defaultInterval = time.Duration(10) * time.Second var started bool +func getScheduleOptions(task *ScheduleTask) []chrono.Option { + if task == nil || task.Type != Interval || task.InitialDelay == "" { + return nil + } + + initialDelay, err := time.ParseDuration(task.InitialDelay) + if err != nil { + log.Warnf("invalid initial delay for task [%s]: %s", task.ID, task.InitialDelay) + return nil + } + if initialDelay <= 0 { + return nil + } + + return []chrono.Option{chrono.WithInitialDelay(initialDelay)} +} + func RunTasks() { started = true Tasks.Range(func(key, value any) bool { @@ -272,7 +290,7 @@ func runTask(task *ScheduleTask) { switch task.Type { case Interval: - task1, err := taskScheduler.ScheduleAtFixedRate(task.Task, util.GetDurationOrDefault(task.Interval, defaultInterval)) + task1, err := taskScheduler.ScheduleAtFixedRate(task.Task, util.GetDurationOrDefault(task.Interval, defaultInterval), getScheduleOptions(task)...) if err != nil { log.Error("failed to scheduled interval task:", task.Type, ",", task.Interval, ",", task.Description) } diff --git a/core/task/task_test.go b/core/task/task_test.go new file mode 100644 index 000000000..678d9d88a --- /dev/null +++ b/core/task/task_test.go @@ -0,0 +1,27 @@ +package task + +import "testing" + +func TestGetScheduleOptionsWithInitialDelay(t *testing.T) { + task := &ScheduleTask{ + Type: Interval, + InitialDelay: "250ms", + } + + options := getScheduleOptions(task) + if len(options) != 1 { + t.Fatalf("expected one schedule option, got %d", len(options)) + } +} + +func TestGetScheduleOptionsSkipsInvalidDelay(t *testing.T) { + task := &ScheduleTask{ + Type: Interval, + InitialDelay: "invalid", + } + + options := getScheduleOptions(task) + if len(options) != 0 { + t.Fatalf("expected no schedule options for invalid delay, got %d", len(options)) + } +} diff --git a/modules/metrics/elastic/elasticsearch.go b/modules/metrics/elastic/elasticsearch.go index e3dae62b5..70d170775 100644 --- a/modules/metrics/elastic/elasticsearch.go +++ b/modules/metrics/elastic/elasticsearch.go @@ -28,6 +28,7 @@ import ( "errors" "fmt" log "github.com/cihub/seelog" + "hash/fnv" "infini.sh/framework/core/config" "infini.sh/framework/core/elastic" "infini.sh/framework/core/event" @@ -156,6 +157,42 @@ func (m *ElasticsearchMetric) shouldCollectNodeAndIndexMetrics(v *elastic.Elasti return true } +func getMetricTaskInitialDelay(clusterID, taskKind, interval string) string { + period := util.GetDurationOrDefault(interval, 10*time.Second) + if period <= 0 { + return "" + } + + hasher := fnv.New64a() + _, _ = hasher.Write([]byte(clusterID)) + _, _ = hasher.Write([]byte(":")) + _, _ = hasher.Write([]byte(taskKind)) + + offset := time.Duration(hasher.Sum64() % uint64(period)) + if offset <= 0 { + return "" + } + return offset.String() +} + +func getMetricTaskTimeout(interval string) time.Duration { + return util.GetDurationOrDefault(interval, 10*time.Second) +} + +func wrapMetricCollectError(clusterName, metricName, endpoint, interval string, err error) error { + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("[%s] collect %s from target cluster endpoint [%s] timed out after %s: %w", clusterName, metricName, endpoint, interval, err) + } + return fmt.Errorf("[%s] collect %s from target cluster endpoint [%s] failed: %w", clusterName, metricName, endpoint, err) +} + +func wrapMetricPersistError(clusterName, metricName string, err error) error { + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("[%s] persist %s to system metrics store timed out after target cluster collection succeeded: %w", clusterName, metricName, err) + } + return fmt.Errorf("[%s] persist %s to system metrics store failed after target cluster collection succeeded: %w", clusterName, metricName, err) +} + func (m *ElasticsearchMetric) Collect() error { if !m.Enabled { return nil @@ -229,7 +266,6 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea log.Debugf("run monitoring task for elasticsearch: %v - %v", k, v.Config.Name) } - var err error monitorConfigs := getMonitorConfigs(v) clusterLevelEnabled := m.shouldCollectClusterLevelMetrics(v) nodeAndIndexEnabled := m.shouldCollectNodeAndIndexMetrics(v) @@ -240,17 +276,18 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea if clusterLevelEnabled && m.ClusterHealth && monitorConfigs.ClusterHealth.Enabled { log.Debugf("collect cluster health: %s, endpoint: %s", k, v.Config.GetAnyEndpoint()) var clusterHealthMetricTask = task.ScheduleTask{ - ID: clusterHealthTaskID, - Description: fmt.Sprintf("monitoring cluster health metric for cluster %s", k), - Type: "interval", - Singleton: true, - Interval: monitorConfigs.ClusterHealth.Interval, + ID: clusterHealthTaskID, + Description: fmt.Sprintf("monitoring cluster health metric for cluster %s", k), + Type: "interval", + Singleton: true, + Interval: monitorConfigs.ClusterHealth.Interval, + InitialDelay: getMetricTaskInitialDelay(k, "cluster_health", monitorConfigs.ClusterHealth.Interval), Task: func(ctx context.Context) { if !v.IsAvailable() { log.Debugf("cluster [%v] is not available, skip collect cluster health metric", v.Config.Name) return } - err = m.CollectClusterHealth(k, v) + err := m.CollectClusterHealth(k, v) if err != nil { log.Error("collect cluster health error: ", err) } @@ -264,17 +301,18 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea if clusterLevelEnabled && m.ClusterStats && monitorConfigs.ClusterStats.Enabled { log.Debugf("collect cluster state: %s, endpoint: %s", k, v.Config.GetAnyEndpoint()) var clusterStatsMetricTask = task.ScheduleTask{ - ID: clusterStatsTaskID, - Description: fmt.Sprintf("monitoring cluster stats metric for cluster %s", k), - Type: "interval", - Singleton: true, - Interval: monitorConfigs.ClusterStats.Interval, + ID: clusterStatsTaskID, + Description: fmt.Sprintf("monitoring cluster stats metric for cluster %s", k), + Type: "interval", + Singleton: true, + Interval: monitorConfigs.ClusterStats.Interval, + InitialDelay: getMetricTaskInitialDelay(k, "cluster_stats", monitorConfigs.ClusterStats.Interval), Task: func(ctx context.Context) { if !v.IsAvailable() { log.Debugf("cluster [%v] is not available, skip collect cluster stats metric", v.Config.Name) return } - err = m.CollectClusterState(k, v) + err := m.CollectClusterState(k, v) if err != nil { log.Error("collect cluster state error: ", err) } @@ -287,11 +325,12 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea //nodes stats if nodeAndIndexEnabled && m.NodeStats && monitorConfigs.NodeStats.Enabled { var nodeStatsMetricTask = task.ScheduleTask{ - ID: nodeStatsTaskID, - Description: fmt.Sprintf("monitoring node stats metric for cluster %s", k), - Type: "interval", - Interval: monitorConfigs.NodeStats.Interval, - Singleton: true, + ID: nodeStatsTaskID, + Description: fmt.Sprintf("monitoring node stats metric for cluster %s", k), + Type: "interval", + Interval: monitorConfigs.NodeStats.Interval, + InitialDelay: getMetricTaskInitialDelay(k, "node_stats", monitorConfigs.NodeStats.Interval), + Singleton: true, Task: func(ctx context.Context) { if !v.IsAvailable() { log.Debugf("cluster [%v] is not available, skip collect node stats metric", v.Config.Name) @@ -302,7 +341,7 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea ) client := elastic.GetClient(k) - shards, err = client.CatShards() + shards, err := client.CatShards() if err != nil { log.Debug(v.Config.Name, " get shards info error: ", err) } @@ -347,7 +386,9 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea if _, ok := shardInfos[nodeID]; ok { shardInfos[nodeID]["indices_count"] = len(indexInfos[nodeID]) } - m.SaveNodeStats(v, nodeID, nodeStats, shardInfos[nodeID]) + if err := m.SaveNodeStats(v, nodeID, nodeStats, shardInfos[nodeID]); err != nil { + log.Error("collect node stats error: ", err) + } } } } else { @@ -363,11 +404,12 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea //indices stats if nodeAndIndexEnabled && (m.AllIndexStats || m.IndexStats) && monitorConfigs.IndexStats.Enabled { var indexStatsMetricTask = task.ScheduleTask{ - ID: indexStatsTaskID, - Description: fmt.Sprintf("monitoring index stats metric for cluster %s", k), - Type: "interval", - Interval: monitorConfigs.IndexStats.Interval, - Singleton: true, + ID: indexStatsTaskID, + Description: fmt.Sprintf("monitoring index stats metric for cluster %s", k), + Type: "interval", + Interval: monitorConfigs.IndexStats.Interval, + InitialDelay: getMetricTaskInitialDelay(k, "index_stats", monitorConfigs.IndexStats.Interval), + Singleton: true, Task: func(ctx context.Context) { if !v.IsAvailable() { log.Debugf("cluster [%v] is not available, skip collect index stats metric", v.Config.Name) @@ -378,7 +420,7 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea ) client := elastic.GetClient(k) - shards, err = client.CatShards() + shards, err := client.CatShards() if err != nil { log.Debug(v.Config.Name, " get shards info error: ", err) //return true @@ -397,10 +439,11 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea shardInfos := map[string][]elastic.CatShardResponse{} if v.IsAvailable() { - indexInfos, err = client.GetIndices("") + fetchedIndexInfos, err := client.GetIndices("") if err != nil { log.Error(v.Config.Name, " get indices info error: ", err) } + indexInfos = fetchedIndexInfos for _, item := range shards { if _, ok := shardInfos[item.Index]; !ok { @@ -414,7 +457,9 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea } if m.AllIndexStats { - m.SaveIndexStats(v, "_all", "_all", indexStats.All.Primaries, indexStats.All.Total, nil, nil) + if err := m.SaveIndexStats(v, "_all", "_all", indexStats.All.Primaries, indexStats.All.Total, nil, nil); err != nil { + log.Error("collect index stats error: ", err) + } } if m.IndexStats { @@ -427,7 +472,9 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea if shardInfos != nil { shardInfo = shardInfos[x] } - m.SaveIndexStats(v, y.Uuid, x, y.Primaries, y.Total, &indexInfo, shardInfo) + if err := m.SaveIndexStats(v, y.Uuid, x, y.Primaries, y.Total, &indexInfo, shardInfo); err != nil { + log.Error("collect index stats error: ", err) + } } } } @@ -481,7 +528,11 @@ func (m *ElasticsearchMetric) SaveNodeStats(v *elastic.ElasticsearchMetadata, no }, } - return m.onSaveEvent(&item) + if err := m.onSaveEvent(&item); err != nil { + return wrapMetricPersistError(v.Config.Name, fmt.Sprintf("node_stats[%s]", nodeID), err) + } + + return nil } func (m *ElasticsearchMetric) SaveIndexStats(v *elastic.ElasticsearchMetadata, indexID, indexName string, primary, total elastic.IndexLevelStats, info *elastic.IndexInfo, shardInfo []elastic.CatShardResponse) error { @@ -519,7 +570,11 @@ func (m *ElasticsearchMetric) SaveIndexStats(v *elastic.ElasticsearchMetadata, i }, } - return m.onSaveEvent(&item) + if err := m.onSaveEvent(&item); err != nil { + return wrapMetricPersistError(v.Config.Name, fmt.Sprintf("index_stats[%s]", indexName), err) + } + + return nil } func (m *ElasticsearchMetric) CollectClusterHealth(k string, v *elastic.ElasticsearchMetadata) error { @@ -530,7 +585,7 @@ func (m *ElasticsearchMetric) CollectClusterHealth(k string, v *elastic.Elastics //add context to control timeout for metric collecting, //since next metric collecting round will be triggered after this one monitorCfg := getMonitorConfigs(v) - du, _ := time.ParseDuration(monitorCfg.ClusterHealth.Interval) + du := getMetricTaskTimeout(monitorCfg.ClusterHealth.Interval) ctx, cancel := context.WithTimeout(context.Background(), du) defer cancel() var ( @@ -539,13 +594,7 @@ func (m *ElasticsearchMetric) CollectClusterHealth(k string, v *elastic.Elastics ) health, err = client.ClusterHealthSpecEndpoint(ctx, v.Config.GetAnyEndpoint(), "indices") if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - // Explicitly handle context deadline exceeded - return fmt.Errorf("[%s] get cluster health context deadline exceeded after %s: %w", v.Config.Name, monitorCfg.ClusterHealth.Interval, err) - } else { - // Handle other errors - return fmt.Errorf("[%s] get cluster health error: %w", v.Config.Name, err) - } + return wrapMetricCollectError(v.Config.Name, "cluster_health", v.Config.GetAnyEndpoint(), monitorCfg.ClusterHealth.Interval, err) } indicesHealth := health.Indices @@ -569,7 +618,7 @@ func (m *ElasticsearchMetric) CollectClusterHealth(k string, v *elastic.Elastics err = m.onSaveEvent(&item) if err != nil { - return fmt.Errorf("[%s] save cluster health error: %w", v.Config.Name, err) + return wrapMetricPersistError(v.Config.Name, "cluster_health", err) } for indexName, healthInfo := range indicesHealth { item = event.Event{ @@ -591,7 +640,7 @@ func (m *ElasticsearchMetric) CollectClusterHealth(k string, v *elastic.Elastics } err = m.onSaveEvent(&item) if err != nil { - return fmt.Errorf("[%s] save index health error: %w", v.Config.Name, err) + return wrapMetricPersistError(v.Config.Name, fmt.Sprintf("index_health[%s]", indexName), err) } } return nil @@ -607,7 +656,7 @@ func (m *ElasticsearchMetric) CollectClusterState(k string, v *elastic.Elasticse //add context to control timeout for metric collecting, //since next metric collecting round will be triggered after this one monitorCfg := getMonitorConfigs(v) - du, _ := time.ParseDuration(monitorCfg.ClusterHealth.Interval) + du := getMetricTaskTimeout(monitorCfg.ClusterStats.Interval) ctx, cancel := context.WithTimeout(context.Background(), du) defer cancel() var err error @@ -617,13 +666,7 @@ func (m *ElasticsearchMetric) CollectClusterState(k string, v *elastic.Elasticse stats, err = client.GetClusterStats(ctx, "") } if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - // Explicitly handle context deadline exceeded - return fmt.Errorf("[%s] get cluster stats context deadline exceeded after %s: %w", v.Config.Name, monitorCfg.ClusterHealth.Interval, err) - } else { - // Handle other errors - return fmt.Errorf("[%s] get cluster stats error: %w", v.Config.Name, err) - } + return wrapMetricCollectError(v.Config.Name, "cluster_stats", v.Config.GetAnyEndpoint(), monitorCfg.ClusterStats.Interval, err) } item := event.Event{ @@ -644,7 +687,11 @@ func (m *ElasticsearchMetric) CollectClusterState(k string, v *elastic.Elasticse }, } - return m.onSaveEvent(&item) + if err := m.onSaveEvent(&item); err != nil { + return wrapMetricPersistError(v.Config.Name, "cluster_stats", err) + } + + return nil } func (m *ElasticsearchMetric) CollectNodeStats() { diff --git a/modules/metrics/elastic/elasticsearch_test.go b/modules/metrics/elastic/elasticsearch_test.go index eead81c34..d3168a9a4 100644 --- a/modules/metrics/elastic/elasticsearch_test.go +++ b/modules/metrics/elastic/elasticsearch_test.go @@ -2,6 +2,7 @@ package elastic import ( "testing" + "time" coreelastic "infini.sh/framework/core/elastic" ) @@ -75,3 +76,33 @@ func TestShouldCollectMetricsAllowsAgentCollectorInAgentMode(t *testing.T) { t.Fatal("expected agent-side collector to run for agent mode clusters") } } + +func TestGetMetricTaskInitialDelayStableAndBounded(t *testing.T) { + first := getMetricTaskInitialDelay("cluster-a", "cluster_health", "10s") + second := getMetricTaskInitialDelay("cluster-a", "cluster_health", "10s") + if first != second { + t.Fatalf("expected stable delay, got %s and %s", first, second) + } + + delay, err := time.ParseDuration(first) + if err != nil { + t.Fatalf("expected parseable delay, got %q: %v", first, err) + } + if delay < 0 || delay >= 10*time.Second { + t.Fatalf("expected delay to be within interval, got %s", delay) + } +} + +func TestGetMetricTaskInitialDelayVariesByTaskKind(t *testing.T) { + healthDelay := getMetricTaskInitialDelay("cluster-a", "cluster_health", "10s") + statsDelay := getMetricTaskInitialDelay("cluster-a", "cluster_stats", "10s") + if healthDelay == statsDelay { + t.Fatalf("expected different metric kinds to spread across interval, both got %s", healthDelay) + } +} + +func TestGetMetricTaskTimeoutFallsBackToDefault(t *testing.T) { + if got := getMetricTaskTimeout("invalid"); got != 10*time.Second { + t.Fatalf("expected default timeout, got %s", got) + } +} From 982d252376b74c916475fb757f2cc0e60c969b2d Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 20 May 2026 13:03:52 +0800 Subject: [PATCH 034/137] improve: use availabels seed host --- core/elastic/actions.go | 34 +++++++--- core/elastic/actions_test.go | 90 +++++++++++++++++++++++++++ modules/elastic/common/config.go | 16 +++-- modules/elastic/common/config_test.go | 27 ++++++++ 4 files changed, 154 insertions(+), 13 deletions(-) create mode 100644 core/elastic/actions_test.go create mode 100644 modules/elastic/common/config_test.go diff --git a/core/elastic/actions.go b/core/elastic/actions.go index f96359f27..5ff110ff2 100644 --- a/core/elastic/actions.go +++ b/core/elastic/actions.go @@ -209,13 +209,8 @@ func (meta *ElasticsearchMetadata) GetActiveEndpoint() string { } func (meta *ElasticsearchMetadata) GetActivePreferredSeedHost() string { - hosts := meta.GetSeedHosts() - if len(hosts) > 0 { - for _, v := range hosts { - if v != "" && IsHostAvailable(v) { - return v - } - } + if host, _ := meta.getAvailableSeedHost(); host != "" { + return host } return meta.Config.Host } @@ -286,6 +281,12 @@ func (meta *ElasticsearchMetadata) GetActiveHosts() int { } func (meta *ElasticsearchMetadata) GetActiveHost() string { + if host, info := meta.getAvailableSeedHost(); host != "" { + if info != nil { + meta.activeHost = info + } + return host + } if meta.activeHost != nil { if meta.activeHost.IsAvailable() { @@ -343,6 +344,25 @@ func (meta *ElasticsearchMetadata) GetActiveHost() string { return hosts[0] } +func (meta *ElasticsearchMetadata) getAvailableSeedHost() (string, *NodeAvailable) { + hosts := meta.GetSeedHosts() + if hosts == nil || len(hosts) == 0 { + return "", nil + } + + for _, host := range hosts { + if host == "" || !IsHostAvailable(host) { + continue + } + if info, ok := GetHostAvailableInfo(host); ok && info != nil && info.IsAvailable() { + return host, info + } + return host, nil + } + + return "", nil +} + func (meta *ElasticsearchMetadata) IsTLS() bool { return meta.GetSchema() == "https" } diff --git a/core/elastic/actions_test.go b/core/elastic/actions_test.go new file mode 100644 index 000000000..f7d2ef830 --- /dev/null +++ b/core/elastic/actions_test.go @@ -0,0 +1,90 @@ +package elastic + +import ( + "testing" + "time" + + "infini.sh/framework/core/orm" +) + +func TestGetActiveHostPrefersAvailableSeedHostOverCachedDiscoveredHost(t *testing.T) { + const ( + clusterID = "docker-mapped-port-cluster" + seedHost = "192.168.3.185:9211" + discoveredHost = "172.18.1.18:9200" + ) + + cfg := &ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: clusterID}, + Name: clusterID, + Host: seedHost, + Hosts: []string{seedHost}, + Enabled: true, + } + cfg.Discovery.Enabled = true + + meta := &ElasticsearchMetadata{ + Config: cfg, + Nodes: &map[string]NodesInfo{ + "node-1": { + Http: struct { + BoundAddress []string `json:"bound_address"` + PublishAddress string `json:"publish_address,omitempty"` + MaxContentLengthInBytes int64 `json:"max_content_length_in_bytes,omitempty"` + }{ + PublishAddress: discoveredHost, + }, + }, + }, + activeHost: &NodeAvailable{Host: discoveredHost, available: true, lastCheck: time.Now()}, + } + + hosts.Store(seedHost, &NodeAvailable{Host: seedHost, ClusterID: clusterID, available: true, lastCheck: time.Now()}) + hosts.Store(discoveredHost, &NodeAvailable{Host: discoveredHost, ClusterID: clusterID, available: true, lastCheck: time.Now()}) + t.Cleanup(func() { + hosts.Delete(seedHost) + hosts.Delete(discoveredHost) + }) + + got := meta.GetActiveHost() + if got != seedHost { + t.Fatalf("expected seed host %q to be preferred over discovered host %q, got %q", seedHost, discoveredHost, got) + } + if meta.activeHost == nil || meta.activeHost.Host != seedHost { + t.Fatalf("expected activeHost to be updated to seed host %q, got %#v", seedHost, meta.activeHost) + } +} + +func TestGetActiveHostFallsBackToCachedDiscoveredHostWhenSeedUnavailable(t *testing.T) { + const ( + clusterID = "docker-discovery-fallback-cluster" + seedHost = "192.168.3.185:9211" + discoveredHost = "172.18.1.18:9200" + ) + + cfg := &ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: clusterID}, + Name: clusterID, + Host: seedHost, + Hosts: []string{seedHost}, + Enabled: true, + } + cfg.Discovery.Enabled = true + + meta := &ElasticsearchMetadata{ + Config: cfg, + activeHost: &NodeAvailable{Host: discoveredHost, available: true, lastCheck: time.Now()}, + } + + hosts.Store(seedHost, &NodeAvailable{Host: seedHost, ClusterID: clusterID, available: false, lastCheck: time.Now()}) + hosts.Store(discoveredHost, &NodeAvailable{Host: discoveredHost, ClusterID: clusterID, available: true, lastCheck: time.Now()}) + t.Cleanup(func() { + hosts.Delete(seedHost) + hosts.Delete(discoveredHost) + }) + + got := meta.GetActiveHost() + if got != discoveredHost { + t.Fatalf("expected discovered host %q when seed host is unavailable, got %q", discoveredHost, got) + } +} diff --git a/modules/elastic/common/config.go b/modules/elastic/common/config.go index 64f610b5d..4c27a29d4 100644 --- a/modules/elastic/common/config.go +++ b/modules/elastic/common/config.go @@ -221,6 +221,9 @@ func InitElasticInstance(esConfig elastic.ElasticsearchConfig) (elastic.API, err log.Warn("elasticsearch ", esConfig.Name, " is not enabled") return nil, nil } + originMeta := elastic.GetMetadata(esConfig.ID) + initHealth := getInitialMetadataHealth(originMeta) + client, err := InitClientWithConfig(esConfig) if err != nil { log.Error("elasticsearch ", esConfig.Name, err) @@ -228,12 +231,6 @@ func InitElasticInstance(esConfig elastic.ElasticsearchConfig) (elastic.API, err } elastic.RegisterInstance(esConfig, client) - originMeta := elastic.GetMetadata(esConfig.ID) - initHealth := true - if originMeta != nil { - initHealth = originMeta.IsAvailable() - } - v := elastic.InitMetadata(&esConfig, initHealth) if v.Health == nil && originMeta != nil { v.Health = originMeta.Health @@ -242,6 +239,13 @@ func InitElasticInstance(esConfig elastic.ElasticsearchConfig) (elastic.API, err return client, err } +func getInitialMetadataHealth(originMeta *elastic.ElasticsearchMetadata) bool { + if originMeta == nil { + return true + } + return originMeta.IsAvailable() +} + func GetBasicAuth(esConfig *elastic.ElasticsearchConfig) (basicAuth *model.BasicAuth, err error) { if esConfig.BasicAuth != nil && esConfig.BasicAuth.Username != "" { basicAuth = esConfig.BasicAuth diff --git a/modules/elastic/common/config_test.go b/modules/elastic/common/config_test.go new file mode 100644 index 000000000..461d5286d --- /dev/null +++ b/modules/elastic/common/config_test.go @@ -0,0 +1,27 @@ +package common + +import ( + "testing" + + "infini.sh/framework/core/elastic" +) + +func TestGetInitialMetadataHealthDefaultsToAvailableForNewCluster(t *testing.T) { + if !getInitialMetadataHealth(nil) { + t.Fatal("expected new cluster metadata to start as available before first health check") + } +} + +func TestGetInitialMetadataHealthKeepsExistingAvailability(t *testing.T) { + meta := &elastic.ElasticsearchMetadata{Config: &elastic.ElasticsearchConfig{Enabled: true}} + meta.Init(false) + + if getInitialMetadataHealth(meta) { + t.Fatal("expected existing unavailable metadata to remain unavailable") + } + + meta.Init(true) + if !getInitialMetadataHealth(meta) { + t.Fatal("expected existing available metadata to remain available") + } +} From 4c9e3f77d45b5b5b12af78e29b2e2f13096d319b Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 21 May 2026 07:31:29 +0800 Subject: [PATCH 035/137] improve: base path with endpoint --- core/config/system.go | 15 +++++++++++++-- core/config/system_test.go | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/core/config/system.go b/core/config/system.go index b34a5ce47..77e649cd5 100755 --- a/core/config/system.go +++ b/core/config/system.go @@ -391,7 +391,7 @@ type S3BucketConfig struct { } func (config *WebAppConfig) GetEndpoint() string { - return fmt.Sprintf("%s://%s", config.GetSchema(), config.NetworkConfig.GetPublishAddr()) + return joinBasePath(fmt.Sprintf("%s://%s", config.GetSchema(), config.NetworkConfig.GetPublishAddr()), config.BasePath) } func (config *WebAppConfig) GetSchema() string { @@ -427,7 +427,18 @@ type APIConfig struct { } func (config *APIConfig) GetEndpoint() string { - return fmt.Sprintf("%s://%s", config.GetSchema(), config.NetworkConfig.GetPublishAddr()) + return joinBasePath(fmt.Sprintf("%s://%s", config.GetSchema(), config.NetworkConfig.GetPublishAddr()), config.BasePath) +} + +func joinBasePath(endpoint, basePath string) string { + basePath = strings.TrimSpace(basePath) + if basePath == "" || basePath == "/" { + return endpoint + } + if !strings.HasPrefix(basePath, "/") { + basePath = "/" + basePath + } + return strings.TrimRight(endpoint, "/") + strings.TrimRight(basePath, "/") } func (config *APIConfig) GetSchema() string { diff --git a/core/config/system_test.go b/core/config/system_test.go index 9224ec52d..7f2d24c5c 100644 --- a/core/config/system_test.go +++ b/core/config/system_test.go @@ -196,3 +196,25 @@ func TestHTTPClientConfig_ValidateProxy(t *testing.T) { } }) } + +func TestGetEndpointIncludesBasePath(t *testing.T) { + t.Run("api endpoint includes normalized base path", func(t *testing.T) { + cfg := APIConfig{ + NetworkConfig: NetworkConfig{Publish: "agent.local:2900"}, + BasePath: "api/v1/", + } + if got := cfg.GetEndpoint(); got != "http://agent.local:2900/api/v1" { + t.Fatalf("expected base path in api endpoint, got %q", got) + } + }) + + t.Run("web endpoint keeps root path unchanged", func(t *testing.T) { + cfg := WebAppConfig{ + NetworkConfig: NetworkConfig{Publish: "console.local:9000"}, + BasePath: "/", + } + if got := cfg.GetEndpoint(); got != "http://console.local:9000" { + t.Fatalf("expected root path to be ignored, got %q", got) + } + }) +} From 733c8892505a6d07a4a1ad7fd807d63f76e8f06f Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 21 May 2026 17:36:56 +0800 Subject: [PATCH 036/137] improve: reduce debug logs for files clean --- core/elastic/domain.go | 4 +- modules/queue/disk_queue/consumer.go | 63 ++++++++- modules/queue/disk_queue/diskqueue_test.go | 142 +++++++++++++++++++++ 3 files changed, 205 insertions(+), 4 deletions(-) diff --git a/core/elastic/domain.go b/core/elastic/domain.go index 4cba058ef..97bb08ca0 100644 --- a/core/elastic/domain.go +++ b/core/elastic/domain.go @@ -488,9 +488,9 @@ type ElasticsearchConfig struct { AllowAccessWhenMasterNotFound bool `json:"allow_access_when_master_not_found,omitempty" config:"allow_access_when_master_not_found"` - BasicAuth *model.BasicAuth `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"` + BasicAuth *model.BasicAuth `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"` // Access token. Easysearch only. - Token ucfg.SecretString `config:"token" json:"token,omitempty" yaml:"token" elastic_mapping:"token:{type:keyword}"` + Token ucfg.SecretString `config:"token" json:"token,omitempty" yaml:"token" elastic_mapping:"token:{type:keyword}"` TrafficControl *struct { Enabled bool `json:"enabled,omitempty" config:"enabled"` diff --git a/modules/queue/disk_queue/consumer.go b/modules/queue/disk_queue/consumer.go index 5811fc1aa..4f4529ae3 100644 --- a/modules/queue/disk_queue/consumer.go +++ b/modules/queue/disk_queue/consumer.go @@ -68,6 +68,29 @@ type Consumer struct { fileLoadCompleted bool } +func (d *Consumer) parkOnEmptyTail(fileName string) error { + if d.readFile != nil { + if err := d.readFile.Close(); err != nil && !util.ContainStr(err.Error(), "already") { + return err + } + } + d.readFile = nil + d.reader = nil + d.fileName = fileName + d.lastFileSize = 0 + d.maxBytesPerFileRead = 0 + d.fileLoadCompleted = false + return nil +} + +func (d *Consumer) waitingForTailFile() bool { + return d.diskQueue != nil && + d.readFile == nil && + d.reader == nil && + d.segment == d.diskQueue.writeSegmentNum && + d.readPos == 0 +} + func (c *Consumer) getFileSize() int64 { var err error readFile, err := os.OpenFile(c.fileName, os.O_RDONLY, 0600) @@ -144,6 +167,22 @@ READ_MSG: // check reader if d.reader == nil { + if d.waitingForTailFile() { + if d.diskQueue.writePos > 0 || util.FileExists(d.fileName) { + err = d.ResetOffset(d.segment, d.readPos) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return messages, false, nil + } + return messages, false, err + } + goto READ_MSG + } + if len(messages) == 0 && d.cCfg.EOFRetryDelayInMs > 0 { + time.Sleep(time.Duration(d.cCfg.EOFRetryDelayInMs) * time.Millisecond) + } + return messages, false, nil + } return messages, false, errors.New("reader is nil") } //read message size @@ -274,8 +313,19 @@ READ_MSG: //can't read ahead before current write file if nextSegment >= d.diskQueue.writeSegmentNum { log.Debugf("need to skip to next file, but next file not exists, current write segment:%v, current read segment:%v", d.diskQueue.writeSegmentNum, d.segment) - d.diskQueue.skipToNextRWFile(false) + err = d.diskQueue.skipToNextRWFile(false) + if err != nil { + return messages, false, err + } d.diskQueue.needSync = true + err = d.ResetOffset(d.diskQueue.writeSegmentNum, 0) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return messages, false, nil + } + return messages, false, err + } + ctx.UpdateNextOffset(d.segment, d.readPos) } else { //let's continue move to next file nextSegment++ @@ -509,6 +559,9 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error { if !exists { //double check, but next file exists if !util.FileExists(fileName) { + if segment == d.diskQueue.writeSegmentNum && readPos == 0 && d.diskQueue.writePos == 0 { + return d.parkOnEmptyTail(fileName) + } if d.mCfg.AutoSkipCorruptFile { nextSegment := d.segment + 1 if nextSegment > d.diskQueue.writeSegmentNum { @@ -518,7 +571,7 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error { d.qCfg.Name, d.queue, d.cCfg.Key(), d.segment, d.readPos, fileName) RETRY_NEXT_FILE: // there are segments in the middle - if nextSegment < d.diskQueue.writeSegmentNum { + if nextSegment <= d.diskQueue.writeSegmentNum { fileName, exists, next_file_exists = SmartGetFileName(d.mCfg, d.queue, nextSegment) if exists || util.FileExists(fileName) { log.Debugf("retry skip to next file: %v, exists", fileName) @@ -532,6 +585,12 @@ func (d *Consumer) ResetOffset(segment, readPos int64) error { goto RETRY_NEXT_FILE } } else { + if d.diskQueue.writePos == 0 { + d.segment = d.diskQueue.writeSegmentNum + d.readPos = 0 + d.diskQueue.UpdateSegmentConsumerInReading(d.ID, d.segment) + return d.parkOnEmptyTail(GetFileName(d.queue, d.segment)) + } return errors.New(fileName + " not found, next segment greater than current write segment") } } else { diff --git a/modules/queue/disk_queue/diskqueue_test.go b/modules/queue/disk_queue/diskqueue_test.go index a0c0e3bb6..b4c522da5 100644 --- a/modules/queue/disk_queue/diskqueue_test.go +++ b/modules/queue/disk_queue/diskqueue_test.go @@ -1,8 +1,15 @@ package queue import ( + "encoding/binary" + "os" + "path/filepath" "testing" "time" + + . "infini.sh/framework/core/env" + "infini.sh/framework/core/global" + corequeue "infini.sh/framework/core/queue" ) func TestGetWriteTimeoutIncludesPayloadAndBacklog(t *testing.T) { @@ -38,3 +45,138 @@ func TestGetWriteTimeoutCapsAtMaximum(t *testing.T) { t.Fatalf("unexpected capped timeout: got %s want %s", timeout, expected) } } + +func TestResetOffsetSkipsMissingSegmentsUpToCurrentWriteSegment(t *testing.T) { + env1 := EmptyEnv() + env1.SystemConfig.PathConfig.Data = t.TempDir() + global.RegisterEnv(env1) + + queueName := "reset-offset-skip" + data := []byte("ok") + fileName := GetFileName(queueName, 2) + if err := os.MkdirAll(filepath.Dir(fileName), 0o755); err != nil { + t.Fatalf("failed to create queue dir: %v", err) + } + file, err := os.Create(fileName) + if err != nil { + t.Fatalf("failed to create segment file: %v", err) + } + if err := binary.Write(file, binary.BigEndian, int32(len(data))); err != nil { + t.Fatalf("failed to write message size: %v", err) + } + if _, err := file.Write(data); err != nil { + t.Fatalf("failed to write message body: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("failed to close segment file: %v", err) + } + + dq := &DiskBasedQueue{ + name: queueName, + cfg: &DiskQueueConfig{AutoSkipCorruptFile: true, MinMsgSize: 1, MaxMsgSize: 1024}, + writeSegmentNum: 2, + writePos: int64(4 + len(data)), + } + consumer := &Consumer{ + ID: "consumer-reset", + diskQueue: dq, + mCfg: dq.cfg, + qCfg: &corequeue.QueueConfig{Name: queueName}, + cCfg: &corequeue.ConsumerConfig{}, + queue: queueName, + } + + if err := consumer.ResetOffset(1, 0); err != nil { + t.Fatalf("expected reset offset to skip to current write segment, got %v", err) + } + if consumer.segment != 2 { + t.Fatalf("expected consumer to move to segment 2, got %d", consumer.segment) + } + if consumer.reader == nil { + t.Fatalf("expected consumer reader to be initialized for segment 2") + } +} + +func TestFetchMessagesRecoversToEmptyTailWithoutRescanningCorruptFile(t *testing.T) { + env1 := EmptyEnv() + env1.SystemConfig.PathConfig.Data = t.TempDir() + global.RegisterEnv(env1) + + queueName := "fetch-empty-tail" + corruptFile := GetFileName(queueName, 1) + if err := os.MkdirAll(filepath.Dir(corruptFile), 0o755); err != nil { + t.Fatalf("failed to create queue dir: %v", err) + } + if err := os.WriteFile(corruptFile, []byte{0x7f, 0xff, 0xff, 0xff}, 0o644); err != nil { + t.Fatalf("failed to write corrupt segment: %v", err) + } + + dq := &DiskBasedQueue{ + name: queueName, + cfg: &DiskQueueConfig{AutoSkipCorruptFile: true, MinMsgSize: 1, MaxMsgSize: 1024}, + writeSegmentNum: 3, + writePos: 0, + } + consumer := &Consumer{ + ID: "consumer-fetch", + diskQueue: dq, + mCfg: dq.cfg, + qCfg: &corequeue.QueueConfig{Name: queueName}, + cCfg: &corequeue.ConsumerConfig{}, + queue: queueName, + } + + if err := consumer.ResetOffset(1, 0); err != nil { + t.Fatalf("failed to initialize consumer: %v", err) + } + + ctx := &corequeue.Context{} + messages, timeout, err := consumer.FetchMessages(ctx, 1) + if err != nil { + t.Fatalf("expected corruption recovery without error, got %v", err) + } + if timeout { + t.Fatalf("did not expect timeout during corruption recovery") + } + if len(messages) != 0 { + t.Fatalf("expected no messages during recovery, got %d", len(messages)) + } + if consumer.segment != dq.writeSegmentNum { + t.Fatalf("expected consumer to park on new tail segment %d, got %d", dq.writeSegmentNum, consumer.segment) + } + if ctx.NextOffset.Segment != dq.writeSegmentNum || ctx.NextOffset.Position != 0 { + t.Fatalf("expected next offset to advance to new tail, got %v", ctx.NextOffset) + } + + payload := []byte("hello") + tailFile := GetFileName(queueName, dq.writeSegmentNum) + file, err := os.Create(tailFile) + if err != nil { + t.Fatalf("failed to create new tail segment: %v", err) + } + if err := binary.Write(file, binary.BigEndian, int32(len(payload))); err != nil { + t.Fatalf("failed to write tail message size: %v", err) + } + if _, err := file.Write(payload); err != nil { + t.Fatalf("failed to write tail message body: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("failed to close tail segment: %v", err) + } + dq.writePos = int64(4 + len(payload)) + + ctx = &corequeue.Context{} + messages, timeout, err = consumer.FetchMessages(ctx, 1) + if err != nil { + t.Fatalf("expected consumer to resume reading on new tail, got %v", err) + } + if timeout { + t.Fatalf("did not expect timeout when new tail data exists") + } + if len(messages) != 1 { + t.Fatalf("expected exactly one message, got %d", len(messages)) + } + if string(messages[0].Data) != "hello" { + t.Fatalf("expected payload %q, got %q", "hello", string(messages[0].Data)) + } +} From 1e7eb1054a01af8b47d9d8097d7818f1fcb2d5f1 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 22 May 2026 06:49:56 +0800 Subject: [PATCH 037/137] improve: instance list with agent stats --- core/api/web.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/api/web.go b/core/api/web.go index a502c4baa..c4617aff8 100755 --- a/core/api/web.go +++ b/core/api/web.go @@ -30,6 +30,7 @@ package api import ( ctx "context" "crypto/tls" + "fmt" "net/http" _ "net/http/pprof" "runtime" @@ -56,6 +57,14 @@ var uiMutex sync.Mutex var bindAddress string +func ServeRegisteredUIRequest(w http.ResponseWriter, req *http.Request) error { + if uiRouter == nil { + return fmt.Errorf("web router is not initialized") + } + uiRouter.ServeHTTP(w, req) + return nil +} + func StopWeb(cfg config.WebAppConfig) { if srv != nil { ctx1, cancel := ctx.WithTimeout(ctx.Background(), 10*time.Second) From 99327692d40b31c5545892851424adbd625f5f68 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 22 May 2026 07:21:02 +0800 Subject: [PATCH 038/137] improve: instance queue/config/task with agent --- core/api/api.go | 29 +++++++++++++++++++++++++++++ core/api/api_test.go | 23 +++++++++++++++++++++++ core/model/instance.go | 13 ++++++++++++- core/model/instance_test.go | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 core/model/instance_test.go diff --git a/core/api/api.go b/core/api/api.go index 4a0540ed4..52167412b 100755 --- a/core/api/api.go +++ b/core/api/api.go @@ -136,6 +136,35 @@ func HandleAPIMethod(method Method, pattern string, handler func(w http.Response l.Unlock() } +func ServeRegisteredAPIRequest(w http.ResponseWriter, req *http.Request) { + localMux := http.NewServeMux() + localRouter := httprouter.New(localMux) + localRouter.NotFound = notfoundHandler + + l.Lock() + defer l.Unlock() + + for pattern, handler := range registeredAPIFuncHandler { + wrapped := handler + for _, f := range filters { + wrapped = f.FilterHttpHandlerFunc(pattern, wrapped) + } + localMux.HandleFunc(pattern, wrapped) + } + + for method, handlers := range registeredAPIMethodHandler { + for pattern, handler := range handlers { + wrapped := handler + for _, f := range filters { + wrapped = f.FilterHttpRouter(pattern, wrapped) + } + localRouter.Handle(method, pattern, wrapped) + } + } + + localRouter.ServeHTTP(w, req) +} + var router = httprouter.New(mux) var mux = http.NewServeMux() diff --git a/core/api/api_test.go b/core/api/api_test.go index 45c7f3129..d0ef493f0 100644 --- a/core/api/api_test.go +++ b/core/api/api_test.go @@ -27,9 +27,12 @@ package api import ( + "fmt" "net/http" "net/http/httptest" "testing" + + httprouter "infini.sh/framework/core/api/router" ) func TestStripPrefix(t *testing.T) { @@ -111,3 +114,23 @@ func TestStripPrefix(t *testing.T) { }) } } + +func TestServeRegisteredAPIRequest(t *testing.T) { + path := fmt.Sprintf("/__copilot_test__/api/%s/:id", t.Name()) + HandleAPIMethod(GET, path, func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte(ps.MustGetParameter("id") + ":" + req.URL.Query().Get("q"))) + }) + + req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("%s/value?q=ok", fmt.Sprintf("/__copilot_test__/api/%s", t.Name())), nil) + recorder := httptest.NewRecorder() + + ServeRegisteredAPIRequest(recorder, req) + + if recorder.Code != http.StatusAccepted { + t.Fatalf("unexpected status: %d", recorder.Code) + } + if recorder.Body.String() != "value:ok" { + t.Fatalf("unexpected body: %s", recorder.Body.String()) + } +} diff --git a/core/model/instance.go b/core/model/instance.go index f9c60b44b..58c0583b8 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -34,6 +34,7 @@ import ( "time" log "github.com/cihub/seelog" + "infini.sh/framework/core/config" "infini.sh/framework/core/env" "infini.sh/framework/core/global" "infini.sh/framework/core/host" @@ -126,6 +127,16 @@ func (inst *Instance) GetVersion() (map[string]interface{}, error) { return nil, fmt.Errorf("unknow agent version") } +func resolveManagedInstanceEndpoint(apiConfig config.APIConfig, webConfig config.WebAppConfig) string { + if apiConfig.Enabled { + return apiConfig.GetEndpoint() + } + if webConfig.Enabled { + return webConfig.GetEndpoint() + } + return apiConfig.GetEndpoint() +} + func GetInstanceInfo() Instance { instance := Instance{} instance.ID = global.Env().SystemConfig.NodeConfig.ID @@ -137,7 +148,7 @@ func GetInstanceInfo() Instance { _, publicIP, _, _ := util.GetPublishNetworkDeviceInfo(global.Env().SystemConfig.NodeConfig.MajorIpPattern) - instance.Endpoint = global.Env().SystemConfig.APIConfig.GetEndpoint() + instance.Endpoint = resolveManagedInstanceEndpoint(global.Env().SystemConfig.APIConfig, global.Env().SystemConfig.WebAppConfig) ips := util.GetLocalIPs() if len(ips) > 0 { diff --git a/core/model/instance_test.go b/core/model/instance_test.go new file mode 100644 index 000000000..be13ca778 --- /dev/null +++ b/core/model/instance_test.go @@ -0,0 +1,33 @@ +package model + +import ( + "testing" + + "infini.sh/framework/core/config" +) + +func TestResolveManagedInstanceEndpoint(t *testing.T) { + t.Run("prefer api endpoint when api is enabled", func(t *testing.T) { + apiConfig := config.APIConfig{Enabled: true} + apiConfig.NetworkConfig.Publish = "127.0.0.1:2900" + webConfig := config.WebAppConfig{Enabled: true} + webConfig.NetworkConfig.Publish = "127.0.0.1:8080" + + endpoint := resolveManagedInstanceEndpoint(apiConfig, webConfig) + if endpoint != "http://127.0.0.1:2900" { + t.Fatalf("unexpected endpoint: %s", endpoint) + } + }) + + t.Run("fallback to web endpoint when api is disabled", func(t *testing.T) { + apiConfig := config.APIConfig{Enabled: false} + apiConfig.NetworkConfig.Publish = "127.0.0.1:2900" + webConfig := config.WebAppConfig{Enabled: true} + webConfig.NetworkConfig.Publish = "127.0.0.1:8080" + + endpoint := resolveManagedInstanceEndpoint(apiConfig, webConfig) + if endpoint != "http://127.0.0.1:8080" { + t.Fatalf("unexpected endpoint: %s", endpoint) + } + }) +} From 52c13b7eee0d4f58276b919525f0b97fe1deb808 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 22 May 2026 11:11:06 +0800 Subject: [PATCH 039/137] improve: instance enroll with web api --- modules/configs/common/config.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/configs/common/config.go b/modules/configs/common/config.go index 03ce8dbe4..ea622b240 100644 --- a/modules/configs/common/config.go +++ b/modules/configs/common/config.go @@ -38,11 +38,12 @@ type AgentConfig struct { } type SetupConfig struct { - DownloadURL string `config:"download_url"` - CACertFile string `config:"ca_cert"` - CAKeyFile string `config:"ca_key"` - ConsoleEndpoint string `config:"console_endpoint"` - Port string `config:"port"` + DownloadURL string `config:"download_url"` + CACertFile string `config:"ca_cert"` + CAKeyFile string `config:"ca_key"` + ConsoleEndpoint string `config:"console_endpoint"` + ReverseChannelEndpoint string `config:"reverse_channel_endpoint"` + Port string `config:"port"` } func GetAgentConfig() *AgentConfig { From ecc8a407ad579a67330d4f5a73f8a2aa91cb1a31 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 22 May 2026 12:35:13 +0800 Subject: [PATCH 040/137] improve: access with token for communicate --- config/generated.go | 8 ++-- core/credential/credential.go | 17 ++++++++ core/credential/domain.go | 65 +++++++++++++++++++++++++++++++ core/model/instance.go | 3 ++ core/model/token.go | 32 ++++++++++++++++ go.mod | 3 ++ modules/configs/client/client.go | 24 +++++++++++- modules/configs/common/domain.go | 16 ++++++++ modules/configs/common/token.go | 66 ++++++++++++++++++++++++++++++++ 9 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 core/model/token.go create mode 100644 modules/configs/common/token.go diff --git a/config/generated.go b/config/generated.go index baf497913..7fe3857a4 100644 --- a/config/generated.go +++ b/config/generated.go @@ -1,11 +1,11 @@ package config -const LastCommitLog = "N/A" +const LastCommitLog = "4c9e3f77d45b5b5b12af78e29b2e2f13096d319b" -const BuildDate = "N/A" +const BuildDate = "2026-05-21T08:45:43Z" -const EOLDate = "N/A" +const EOLDate = "2023-12-31T10:10:10Z" -const Version = "0.0.1-SNAPSHOT" +const Version = "1.0.0_SNAPSHOT" const BuildNumber = "001" diff --git a/core/credential/credential.go b/core/credential/credential.go index 62e17a4e6..c96970556 100644 --- a/core/credential/credential.go +++ b/core/credential/credential.go @@ -69,10 +69,13 @@ func (cred *Credential) Encode() error { switch cred.Type { case BasicAuth: return encodeBasicAuth(cred) + case Token: + return encodeToken(cred) default: return fmt.Errorf("unkonow credential type [%s]", cred.Type) } } + func (cred *Credential) DecodeBasicAuth() (*model.BasicAuth, error) { var dv interface{} dv, err := cred.Decode() @@ -86,10 +89,23 @@ func (cred *Credential) DecodeBasicAuth() (*model.BasicAuth, error) { return nil, fmt.Errorf("unkonow credential type [%s]", cred.Type) } +func (cred *Credential) DecodeToken() (string, error) { + dv, err := cred.Decode() + if err != nil { + return "", err + } + if token, ok := dv.(model.Token); ok { + return token.Value, nil + } + return "", fmt.Errorf("unkonow credential type [%s]", cred.Type) +} + func (cred *Credential) Decode() (interface{}, error) { switch cred.Type { case BasicAuth: return decodeBasicAuth(cred) + case Token: + return decodeToken(cred) default: return nil, fmt.Errorf("unkonow credential type [%s]", cred.Type) } @@ -97,4 +113,5 @@ func (cred *Credential) Decode() (interface{}, error) { const ( BasicAuth string = "basic_auth" + Token string = "token" ) diff --git a/core/credential/domain.go b/core/credential/domain.go index 2dbb05f4c..63fcc99e4 100644 --- a/core/credential/domain.go +++ b/core/credential/domain.go @@ -157,6 +157,71 @@ func decodeBasicAuth(cred *Credential) (basicAuth model.BasicAuth, err error) { return } +func encodeToken(cred *Credential) error { + params, ok := cred.Payload[cred.Type].(map[string]interface{}) + if !ok { + return fmt.Errorf("wrong credential parameters for type [%s], expect a map", cred.Type) + } + value, ok := params["value"].(string) + if !ok { + return fmt.Errorf("wrong credential parameters value for type [%s], expect a string", cred.Type) + } + if value == "" { + return fmt.Errorf("credential parameters value can not be empty") + } + + secret, err := GetOrInitSecret() + if err != nil { + return err + } + encodeBytes, salt, err := util.AesGcmEncrypt([]byte(value), secret) + if err != nil { + return fmt.Errorf("encrypt token value error: %w", err) + } + cred.Encrypt.Type = "AES" + cred.Encrypt.Params = map[string]interface{}{ + "salt": string(salt), + } + params["value"] = string(encodeBytes) + cred.Payload[cred.Type] = params + return nil +} + +func decodeToken(cred *Credential) (token model.Token, err error) { + params, ok := cred.Payload[cred.Type].(map[string]interface{}) + if !ok { + err = fmt.Errorf("wrong credential parameters for type [%s], expect a map", cred.Type) + return + } + value, ok := params["value"].(string) + if !ok { + err = fmt.Errorf("wrong credential parameters value for type [%s], expect a string", cred.Type) + return + } + if value == "" { + err = fmt.Errorf("credential parameters value can not be empty") + return + } + salt, ok := cred.Encrypt.Params["salt"].(string) + if !ok { + err = fmt.Errorf("credential encrypt parameters salt can not be empty") + return + } + secret := cred.secret + if secret == nil { + secret, err = GetOrInitSecret() + if err != nil { + return token, err + } + } + plaintext, err := util.AesGcmDecrypt([]byte(value), secret, []byte(salt)) + if err != nil { + return token, err + } + token.Value = string(plaintext) + return +} + type ChangeEvent func(credentials *Credential) var changeEvents []ChangeEvent diff --git a/core/model/instance.go b/core/model/instance.go index 58c0583b8..6b9e1abfe 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -57,6 +57,9 @@ type Instance struct { BasicAuth *BasicAuth `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"` + ManagerCredentialID string `json:"manager_credential_id,omitempty" elastic_mapping:"manager_credential_id:{type:keyword}"` + AccessCredentialID string `json:"access_credential_id,omitempty" elastic_mapping:"access_credential_id:{type:keyword}"` + Labels map[string]string `json:"labels,omitempty" elastic_mapping:"labels:{type:object}"` Tags []string `json:"tags,omitempty"` diff --git a/core/model/token.go b/core/model/token.go new file mode 100644 index 000000000..25fc4c74d --- /dev/null +++ b/core/model/token.go @@ -0,0 +1,32 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package model + +type Token struct { + Value string `json:"value,omitempty" config:"value"` +} diff --git a/go.mod b/go.mod index ad609f202..64788225d 100644 --- a/go.mod +++ b/go.mod @@ -83,6 +83,7 @@ require ( gopkg.in/hjson/hjson-go.v3 v3.3.0 gopkg.in/square/go-jose.v2 v2.6.0 gopkg.in/yaml.v2 v2.4.0 + infini.sh/license v0.0.0-00010101000000-000000000000 k8s.io/api v0.32.3 k8s.io/apimachinery v0.32.3 ) @@ -170,3 +171,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) + +replace infini.sh/license => ../license diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index f863434b2..2925c3b1a 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -77,11 +77,25 @@ func ConnectToManager() error { } info := model.GetInstanceInfo() + registerReq := common.InstanceRegisterRequest{ + Client: info, + } + if info.Application.Name == "agent" { + accessToken, err := common.EnsureTokenInKeystore(common.AgentAccessTokenKeystoreKey) + if err != nil { + return err + } + registerReq.AccessToken = &common.RegisterToken{ + Name: fmt.Sprintf("%s reverse access token", info.ID), + Description: fmt.Sprintf("Console to Agent access token for instance %s", info.ID), + Value: accessToken, + } + } req := util.Request{Method: util.Verb_POST} req.ContentType = "application/json" req.Path = common.REGISTER_API - req.Body = util.MustToJSONBytes(info) + req.Body = util.MustToJSONBytes(registerReq) server, res, err := submitRequestToManager(&req) if err == nil && server != "" { @@ -103,7 +117,13 @@ func submitRequestToManager(req *util.Request) (string, *util.Result, error) { var err error var res *util.Result cfg := global.Env().SystemConfig.Configs - if cfg.ManagerConfig.BasicAuth.Username != "" { + token, err := common.LoadTokenFromKeystore(common.ManagerTokenKeystoreKey) + if err != nil { + return "", nil, err + } + if token != "" { + req.AddHeader("Authorization", "Bearer "+token) + } else if cfg.ManagerConfig.BasicAuth.Username != "" { req.SetBasicAuth(cfg.ManagerConfig.BasicAuth.Username, cfg.ManagerConfig.BasicAuth.Password.Get()) } for _, server := range cfg.Servers { diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index 232c1fc52..6bf0914d2 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -32,6 +32,22 @@ import "infini.sh/framework/core/model" const REGISTER_API = "/instance/_register" const SYNC_API = "/configs/_sync" +const ( + ManagerTokenKeystoreKey = "configs_manager_token" + AgentAccessTokenKeystoreKey = "agent_reverse_access_token" +) + +type RegisterToken struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Value string `json:"value,omitempty"` +} + +type InstanceRegisterRequest struct { + Client model.Instance `json:"client"` + AccessToken *RegisterToken `json:"access_token,omitempty"` +} + type ConfigFile struct { Name string `json:"name,omitempty"` Location string `json:"location,omitempty"` diff --git a/modules/configs/common/token.go b/modules/configs/common/token.go new file mode 100644 index 000000000..7f7e634cc --- /dev/null +++ b/modules/configs/common/token.go @@ -0,0 +1,66 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package common + +import ( + "strings" + + "infini.sh/framework/core/keystore" + "infini.sh/framework/core/util" + keystore2 "infini.sh/framework/lib/keystore" +) + +func LoadTokenFromKeystore(key string) (string, error) { + value, err := keystore.GetValue(key) + if err == keystore2.ErrKeyDoesntExists { + return "", nil + } + if err != nil { + return "", err + } + return strings.TrimSpace(string(value)), nil +} + +func SaveTokenToKeystore(key, value string) error { + return keystore.SetValue(key, util.UnsafeStringToBytes(strings.TrimSpace(value))) +} + +func EnsureTokenInKeystore(key string) (string, error) { + value, err := LoadTokenFromKeystore(key) + if err != nil { + return "", err + } + if value != "" { + return value, nil + } + value = util.GenerateRandomString(48) + if err := SaveTokenToKeystore(key, value); err != nil { + return "", err + } + return value, nil +} From 74a3102e1e3d604a3c673d7695660925a5fbabdc Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 22 May 2026 13:38:16 +0800 Subject: [PATCH 041/137] improve: add instance with info --- modules/configs/common/domain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index 6bf0914d2..2c4a7f0ed 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -34,7 +34,7 @@ const SYNC_API = "/configs/_sync" const ( ManagerTokenKeystoreKey = "configs_manager_token" - AgentAccessTokenKeystoreKey = "agent_reverse_access_token" + AgentAccessTokenKeystoreKey = "agent_access_token" ) type RegisterToken struct { From 15042b118742bc6603b520e187a27bfe270ccddf Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 22 May 2026 14:14:35 +0800 Subject: [PATCH 042/137] fix: entry reload callback error --- core/config/fs_watcher.go | 28 +++++++++++-------- core/config/fs_watcher_test.go | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 12 deletions(-) diff --git a/core/config/fs_watcher.go b/core/config/fs_watcher.go index a81408d36..873dd0700 100644 --- a/core/config/fs_watcher.go +++ b/core/config/fs_watcher.go @@ -69,19 +69,21 @@ func dispatchConfigChangeEvent(ev fsnotify.Event, watcherCallbacks []CallbackFun cfg := loadConfigFile(ev.Name) if cfg != nil { - for k, v := range sectionCallbacks { - if cfg.HasField(k) { - currentCfg, err := cfg.Child(k, -1) - if err != nil { - log.Error(err) - continue - } - previousCfg, _ := latestConfig[k] - for _, f := range v { - f(previousCfg, currentCfg) - } - latestConfig[k] = currentCfg + for _, k := range sectionCallbackOrder { + callbacks, ok := sectionCallbacks[k] + if !ok || !cfg.HasField(k) { + continue + } + currentCfg, err := cfg.Child(k, -1) + if err != nil { + log.Error(err) + continue + } + previousCfg, _ := latestConfig[k] + for _, f := range callbacks { + f(previousCfg, currentCfg) } + latestConfig[k] = currentCfg } } @@ -250,6 +252,7 @@ func StopWatchers() { } var sectionCallbacks = map[string][]func(pCfg, cCfg *Config){} +var sectionCallbackOrder = []string{} var configCallbacks = []func(fsnotify.Event){} var cfgLocker = sync.RWMutex{} @@ -264,6 +267,7 @@ func NotifyOnConfigSectionChange(configKey string, f func(pCfg, cCfg *Config)) { if !ok { v = []func(pCfg, cCfg *Config){} sectionCallbacks[configKey] = v + sectionCallbackOrder = append(sectionCallbackOrder, configKey) } v = append(v, f) sectionCallbacks[configKey] = v diff --git a/core/config/fs_watcher_test.go b/core/config/fs_watcher_test.go index 29d4559d4..c1ddf1f75 100644 --- a/core/config/fs_watcher_test.go +++ b/core/config/fs_watcher_test.go @@ -17,13 +17,16 @@ func TestDispatchConfigChangeEventRunsSectionCallbacksBeforeGenericCallbacks(t * } previousSections := sectionCallbacks + previousOrder := sectionCallbackOrder previousConfigs := configCallbacks previousLatest := latestConfig sectionCallbacks = map[string][]func(pCfg, cCfg *Config){} + sectionCallbackOrder = nil configCallbacks = nil latestConfig = map[string]*Config{} t.Cleanup(func() { sectionCallbacks = previousSections + sectionCallbackOrder = previousOrder configCallbacks = previousConfigs latestConfig = previousLatest }) @@ -45,3 +48,50 @@ func TestDispatchConfigChangeEventRunsSectionCallbacksBeforeGenericCallbacks(t * t.Fatalf("expected section callback before generic callback, got %v", order) } } + +func TestDispatchConfigChangeEventRunsSectionCallbacksInRegistrationOrder(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "gateway.yml") + content := []byte("flow:\n - name: flow-1\nrouter:\n - name: router-1\nentry:\n - name: entry-1\n") + if err := os.WriteFile(file, content, 0o644); err != nil { + t.Fatalf("write config file: %v", err) + } + + previousSections := sectionCallbacks + previousOrder := sectionCallbackOrder + previousConfigs := configCallbacks + previousLatest := latestConfig + sectionCallbacks = map[string][]func(pCfg, cCfg *Config){} + sectionCallbackOrder = nil + configCallbacks = nil + latestConfig = map[string]*Config{} + t.Cleanup(func() { + sectionCallbacks = previousSections + sectionCallbackOrder = previousOrder + configCallbacks = previousConfigs + latestConfig = previousLatest + }) + + var order []string + NotifyOnConfigSectionChange("flow", func(pCfg, cCfg *Config) { + order = append(order, "flow") + }) + NotifyOnConfigSectionChange("router", func(pCfg, cCfg *Config) { + order = append(order, "router") + }) + NotifyOnConfigSectionChange("entry", func(pCfg, cCfg *Config) { + order = append(order, "entry") + }) + + dispatchConfigChangeEvent(fsnotify.Event{Name: file, Op: fsnotify.Write}, nil) + + expected := []string{"flow", "router", "entry"} + if len(order) != len(expected) { + t.Fatalf("expected %d callbacks, got %d (%v)", len(expected), len(order), order) + } + for i, want := range expected { + if order[i] != want { + t.Fatalf("expected callback order %v, got %v", expected, order) + } + } +} From 0bdc1d93a2d3c0b2688cd891002dc9c69d691ff4 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 22 May 2026 18:34:35 +0800 Subject: [PATCH 043/137] improve: update websocket to 8mb --- core/api/websocket/conn.go | 6 ++++-- core/api/websocket/hub.go | 8 ++++++++ core/api/websocket/hub_test.go | 36 ++++++++++++++++++++++++++++++++++ core/config/system.go | 1 + 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 core/api/websocket/hub_test.go diff --git a/core/api/websocket/conn.go b/core/api/websocket/conn.go index 56e43c57f..77504d0c0 100755 --- a/core/api/websocket/conn.go +++ b/core/api/websocket/conn.go @@ -48,10 +48,12 @@ const ( // Send pings to peer with this period. Must be less than pongWait. pingPeriod = (pongWait * 9) / 10 - // Maximum message size allowed from peer. - maxMessageSize = 512 + // Default maximum message size allowed from peer. + defaultMaxMessageSize int64 = 8 * 1024 * 1024 ) +var maxMessageSize int64 = defaultMaxMessageSize + var upgrader = websocket.Upgrader{ ReadBufferSize: 1024, WriteBufferSize: 1024, diff --git a/core/api/websocket/hub.go b/core/api/websocket/hub.go index 068c3c473..386e5801c 100755 --- a/core/api/websocket/hub.go +++ b/core/api/websocket/hub.go @@ -87,6 +87,7 @@ func (h *Hub) registerHandlers() { // InitWebSocket start websocket func InitWebSocket(cfg config.WebsocketConfig) { + maxMessageSize = resolveMaxMessageSize(cfg) if cfg.SkipHostVerify { upgrader.CheckOrigin = func(r *http.Request) bool { return true @@ -123,6 +124,13 @@ func InitWebSocket(cfg config.WebsocketConfig) { } +func resolveMaxMessageSize(cfg config.WebsocketConfig) int64 { + if cfg.MaxMessageSizeBytes > 0 { + return cfg.MaxMessageSizeBytes + } + return defaultMaxMessageSize +} + // HandleWebSocketCommand used to register command and handler func HandleWebSocketCommand(cmd, usage string, handler func(c *WebsocketConnection, array []string)) { cmd = strings.ToLower(strings.TrimSpace(cmd)) diff --git a/core/api/websocket/hub_test.go b/core/api/websocket/hub_test.go new file mode 100644 index 000000000..380fd1faf --- /dev/null +++ b/core/api/websocket/hub_test.go @@ -0,0 +1,36 @@ +package websocket + +import ( + "testing" + + "infini.sh/framework/core/config" +) + +func TestResolveMaxMessageSize(t *testing.T) { + testCases := []struct { + name string + cfg config.WebsocketConfig + expect int64 + }{ + { + name: "default", + cfg: config.WebsocketConfig{}, + expect: defaultMaxMessageSize, + }, + { + name: "custom", + cfg: config.WebsocketConfig{ + MaxMessageSizeBytes: 1024, + }, + expect: 1024, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if actual := resolveMaxMessageSize(tc.cfg); actual != tc.expect { + t.Fatalf("unexpected websocket message limit: got %d want %d", actual, tc.expect) + } + }) + } +} diff --git a/core/config/system.go b/core/config/system.go index 77e649cd5..687860379 100755 --- a/core/config/system.go +++ b/core/config/system.go @@ -498,6 +498,7 @@ type WebsocketConfig struct { EchoWelcomeMessageOnConnect bool `config:"echo_welcome_message_on_connect"` EchoLoggingConfigOnConnect bool `config:"echo_logging_config_on_connect"` BasePath string `config:"base_path"` + MaxMessageSizeBytes int64 `config:"max_message_size_bytes"` PermittedHosts []string `config:"permitted_hosts"` SkipHostVerify bool `config:"skip_host_verify"` } From c3a513fa55b1f623637dcd342876b50b822044f8 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 23 May 2026 07:39:36 +0800 Subject: [PATCH 044/137] improve: refactor for the code --- core/api/security.go | 102 +++++++++ core/api/security_test.go | 106 ++++++++++ .../passwordchallenge/password_challenge.go | 164 +++++++++++++++ .../password_challenge_test.go | 29 +++ core/security/replay/replay.go | 199 ++++++++++++++++++ core/security/replay/replay_test.go | 59 ++++++ 6 files changed, 659 insertions(+) create mode 100644 core/api/security.go create mode 100644 core/api/security_test.go create mode 100644 core/security/passwordchallenge/password_challenge.go create mode 100644 core/security/passwordchallenge/password_challenge_test.go create mode 100644 core/security/replay/replay.go create mode 100644 core/security/replay/replay_test.go diff --git a/core/api/security.go b/core/api/security.go new file mode 100644 index 000000000..67226a018 --- /dev/null +++ b/core/api/security.go @@ -0,0 +1,102 @@ +package api + +import ( + "net/http" + "strings" + + httprouter "infini.sh/framework/core/api/router" + replaysecurity "infini.sh/framework/core/security/replay" +) + +type SecureTransportOptions struct { + TrustForwardHeaders bool +} + +func RequestUsesSecureTransport(req *http.Request, options ...SecureTransportOptions) bool { + if req == nil { + return false + } + if req.TLS != nil { + return true + } + + resolved := resolveSecureTransportOptions(options) + if !resolved.TrustForwardHeaders { + return false + } + + for _, header := range []string{"X-Forwarded-Proto", "X-Forwarded-Protocol", "X-Url-Scheme"} { + if headerIndicatesHTTPS(req.Header.Get(header)) { + return true + } + } + + if strings.EqualFold(strings.TrimSpace(req.Header.Get("X-Forwarded-Ssl")), "on") { + return true + } + + return forwardedHeaderIndicatesHTTPS(req.Header.Get("Forwarded")) +} + +func (handler Handler) RequireSecureTransport(h httprouter.Handle, options ...SecureTransportOptions) httprouter.Handle { + resolved := resolveSecureTransportOptions(options) + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + if !RequestUsesSecureTransport(r, resolved) { + handler.WriteError(w, "sensitive endpoints require HTTPS or a trusted HTTPS reverse proxy", http.StatusUpgradeRequired) + return + } + h(w, r, ps) + } +} + +func RequireSecureTransport(h httprouter.Handle, options ...SecureTransportOptions) httprouter.Handle { + return Handler{}.RequireSecureTransport(h, options...) +} + +func (handler Handler) RequireReplayProtection(h httprouter.Handle) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + if err := replaysecurity.ValidateAndConsumeReplayNonce(r); err != nil { + handler.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + h(w, r, ps) + } +} + +func RequireReplayProtection(h httprouter.Handle) httprouter.Handle { + return Handler{}.RequireReplayProtection(h) +} + +func resolveSecureTransportOptions(options []SecureTransportOptions) SecureTransportOptions { + if len(options) == 0 { + return SecureTransportOptions{} + } + return options[0] +} + +func headerIndicatesHTTPS(value string) bool { + if value == "" { + return false + } + first := strings.TrimSpace(strings.Split(value, ",")[0]) + return strings.EqualFold(first, "https") +} + +func forwardedHeaderIndicatesHTTPS(value string) bool { + if value == "" { + return false + } + + for _, forwardedValue := range strings.Split(value, ",") { + for _, token := range strings.Split(forwardedValue, ";") { + parts := strings.SplitN(strings.TrimSpace(token), "=", 2) + if len(parts) != 2 || !strings.EqualFold(parts[0], "proto") { + continue + } + proto := strings.Trim(parts[1], "\"") + return strings.EqualFold(proto, "https") + } + } + + return false +} diff --git a/core/api/security_test.go b/core/api/security_test.go new file mode 100644 index 000000000..8b688b63a --- /dev/null +++ b/core/api/security_test.go @@ -0,0 +1,106 @@ +package api + +import ( + "crypto/tls" + "net/http" + "net/http/httptest" + "testing" + + httprouter "infini.sh/framework/core/api/router" + replaysecurity "infini.sh/framework/core/security/replay" +) + +func TestRequestUsesSecureTransport(t *testing.T) { + tests := []struct { + name string + setup func(req *http.Request) + options []SecureTransportOptions + secure bool + }{ + { + name: "tls request", + setup: func(req *http.Request) { + req.TLS = &tls.ConnectionState{} + }, + secure: true, + }, + { + name: "forwarded proto requires opt in", + setup: func(req *http.Request) { + req.Header.Set("X-Forwarded-Proto", "https") + }, + secure: false, + }, + { + name: "forwarded proto trusted when enabled", + setup: func(req *http.Request) { + req.Header.Set("X-Forwarded-Proto", "https") + }, + options: []SecureTransportOptions{{TrustForwardHeaders: true}}, + secure: true, + }, + { + name: "plain http", + setup: func(req *http.Request) {}, + secure: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "http://console.local/account/login", nil) + tt.setup(req) + + if RequestUsesSecureTransport(req, tt.options...) != tt.secure { + t.Fatalf("expected secure=%v", tt.secure) + } + }) + } +} + +func TestRequireSecureTransport(t *testing.T) { + handler := Handler{} + called := false + protected := handler.RequireSecureTransport(func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + called = true + w.WriteHeader(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodPost, "http://console.local/account/login", nil) + resp := httptest.NewRecorder() + + protected(resp, req, nil) + + if called { + t.Fatal("expected insecure request to be blocked") + } + if resp.Code != http.StatusUpgradeRequired { + t.Fatalf("expected status %d, got %d", http.StatusUpgradeRequired, resp.Code) + } +} + +func TestRequireReplayProtection(t *testing.T) { + handler := Handler{} + req := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) + nonce, _, err := replaysecurity.IssueReplayNonce(req, http.MethodPost, "/account/login") + if err != nil { + t.Fatalf("issue replay nonce: %v", err) + } + req.Header.Set(replaysecurity.HeaderName, nonce) + + called := false + protected := handler.RequireReplayProtection(func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + called = true + w.WriteHeader(http.StatusOK) + }) + resp := httptest.NewRecorder() + + protected(resp, req, nil) + + if !called { + t.Fatal("expected replay-protected handler to run") + } + if resp.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, resp.Code) + } +} diff --git a/core/security/passwordchallenge/password_challenge.go b/core/security/passwordchallenge/password_challenge.go new file mode 100644 index 000000000..461914b50 --- /dev/null +++ b/core/security/passwordchallenge/password_challenge.go @@ -0,0 +1,164 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package passwordchallenge + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "strings" + "sync" + "time" + + "golang.org/x/crypto/pbkdf2" + "infini.sh/framework/core/util" +) + +const ( + Method = "challenge" + Algorithm = "PBKDF2-SHA256" + Iterations = 120000 + keyLength = 32 + DefaultTTL = 5 * time.Minute +) + +type Challenge struct { + ID string + Subject string + Nonce string + ExpireAt time.Time +} + +type StoreOptions struct { + TTL time.Duration +} + +type Store struct { + mu sync.Mutex + ttl time.Duration + challenges map[string]Challenge +} + +var defaultStore = NewStore(StoreOptions{}) + +func NewStore(options StoreOptions) *Store { + ttl := options.TTL + if ttl <= 0 { + ttl = DefaultTTL + } + return &Store{ + ttl: ttl, + challenges: map[string]Challenge{}, + } +} + +func DeriveVerifier(password, salt string) (string, error) { + if password == "" { + return "", errors.New("password is empty") + } + if salt == "" { + return "", errors.New("password salt is empty") + } + key := pbkdf2.Key([]byte(password), []byte(salt), Iterations, keyLength, sha256.New) + return hex.EncodeToString(key), nil +} + +func BuildProof(verifier, subject, challengeID, nonce string) (string, error) { + key, err := hex.DecodeString(verifier) + if err != nil { + return "", err + } + mac := hmac.New(sha256.New, key) + mac.Write([]byte(subject)) + mac.Write([]byte(":")) + mac.Write([]byte(challengeID)) + mac.Write([]byte(":")) + mac.Write([]byte(nonce)) + return hex.EncodeToString(mac.Sum(nil)), nil +} + +func VerifyProof(verifier, subject, challengeID, nonce, proof string) bool { + expected, err := BuildProof(verifier, subject, challengeID, nonce) + if err != nil { + return false + } + expectedBytes, err := hex.DecodeString(expected) + if err != nil { + return false + } + proofBytes, err := hex.DecodeString(strings.ToLower(proof)) + if err != nil { + return false + } + return hmac.Equal(expectedBytes, proofBytes) +} + +func New(subject string) Challenge { + return defaultStore.New(subject) +} + +func Consume(challengeID, subject string) (Challenge, error) { + return defaultStore.Consume(challengeID, subject) +} + +func (store *Store) New(subject string) Challenge { + now := time.Now() + store.mu.Lock() + defer store.mu.Unlock() + + for id, challenge := range store.challenges { + if challenge.ExpireAt.Before(now) { + delete(store.challenges, id) + } + } + + challenge := Challenge{ + ID: util.GenerateSecureString(32), + Subject: subject, + Nonce: util.GenerateSecureString(32), + ExpireAt: now.Add(store.ttl), + } + store.challenges[challenge.ID] = challenge + return challenge +} + +func (store *Store) Consume(challengeID, subject string) (Challenge, error) { + store.mu.Lock() + defer store.mu.Unlock() + + challenge, ok := store.challenges[challengeID] + if !ok { + return Challenge{}, errors.New("login challenge is invalid") + } + delete(store.challenges, challengeID) + + if challenge.ExpireAt.Before(time.Now()) { + return Challenge{}, errors.New("login challenge expired") + } + if challenge.Subject != subject { + return Challenge{}, errors.New("login challenge does not match user") + } + return challenge, nil +} diff --git a/core/security/passwordchallenge/password_challenge_test.go b/core/security/passwordchallenge/password_challenge_test.go new file mode 100644 index 000000000..ed9eff938 --- /dev/null +++ b/core/security/passwordchallenge/password_challenge_test.go @@ -0,0 +1,29 @@ +package passwordchallenge + +import "testing" + +func TestPasswordChallengeProofRoundTrip(t *testing.T) { + verifier, err := DeriveVerifier("admin", "salt-123") + if err != nil { + t.Fatalf("derive verifier: %v", err) + } + + challenge := New("admin") + proof, err := BuildProof(verifier, "admin", challenge.ID, challenge.Nonce) + if err != nil { + t.Fatalf("build proof: %v", err) + } + + if !VerifyProof(verifier, "admin", challenge.ID, challenge.Nonce, proof) { + t.Fatal("expected password proof to validate") + } +} + +func TestConsumeRejectsWrongSubject(t *testing.T) { + store := NewStore(StoreOptions{}) + challenge := store.New("admin") + + if _, err := store.Consume(challenge.ID, "guest"); err == nil { + t.Fatal("expected challenge subject mismatch to fail") + } +} diff --git a/core/security/replay/replay.go b/core/security/replay/replay.go new file mode 100644 index 000000000..1619f0820 --- /dev/null +++ b/core/security/replay/replay.go @@ -0,0 +1,199 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replay + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + pathutil "path" + "strings" + "sync" + "time" + + "infini.sh/framework/core/util" +) + +const ( + HeaderName = "X-Request-Nonce" + DefaultTTL = 30 * time.Second +) + +type SubjectExtractor func(r *http.Request) string + +type StoreOptions struct { + TTL time.Duration + SubjectExtractor SubjectExtractor +} + +type nonceRecord struct { + Subject string + Method string + Path string + ExpiresAt time.Time +} + +type Store struct { + mu sync.Mutex + ttl time.Duration + subjectExtractor SubjectExtractor + records map[string]nonceRecord +} + +var defaultStore = NewStore(StoreOptions{}) + +func NewStore(options StoreOptions) *Store { + ttl := options.TTL + if ttl <= 0 { + ttl = DefaultTTL + } + extractor := options.SubjectExtractor + if extractor == nil { + extractor = DefaultSubjectExtractor + } + return &Store{ + ttl: ttl, + subjectExtractor: extractor, + records: map[string]nonceRecord{}, + } +} + +func IssueReplayNonce(r *http.Request, method, requestPath string) (string, time.Duration, error) { + return defaultStore.IssueReplayNonce(r, method, requestPath) +} + +func ValidateAndConsumeReplayNonce(r *http.Request) error { + return defaultStore.ValidateAndConsumeReplayNonce(r) +} + +func DefaultSubjectExtractor(r *http.Request) string { + if r == nil { + return "anonymous" + } + authorizationHeader := strings.TrimSpace(r.Header.Get("Authorization")) + if authorizationHeader == "" { + return "anonymous" + } + sum := sha256.Sum256([]byte(authorizationHeader)) + return hex.EncodeToString(sum[:]) +} + +func (store *Store) IssueReplayNonce(r *http.Request, method, requestPath string) (string, time.Duration, error) { + normalizedMethod, normalizedPath, err := normalizeScope(method, requestPath) + if err != nil { + return "", 0, err + } + + nonce := util.GenerateSecureString(32) + if nonce == "" { + return "", 0, fmt.Errorf("failed to generate replay nonce") + } + + subject := store.extractSubject(r) + expiresAt := time.Now().Add(store.ttl) + + store.mu.Lock() + defer store.mu.Unlock() + store.cleanupExpiredLocked(time.Now()) + store.records[nonce] = nonceRecord{ + Subject: subject, + Method: normalizedMethod, + Path: normalizedPath, + ExpiresAt: expiresAt, + } + return nonce, store.ttl, nil +} + +func (store *Store) ValidateAndConsumeReplayNonce(r *http.Request) error { + if r == nil { + return fmt.Errorf("request can not be nil") + } + + nonce := strings.TrimSpace(r.Header.Get(HeaderName)) + if nonce == "" { + return fmt.Errorf("missing replay nonce") + } + + subject := store.extractSubject(r) + method, requestPath, err := normalizeScope(r.Method, r.URL.Path) + if err != nil { + return err + } + + now := time.Now() + store.mu.Lock() + defer store.mu.Unlock() + store.cleanupExpiredLocked(now) + + record, ok := store.records[nonce] + if !ok { + return fmt.Errorf("replay nonce is invalid or expired") + } + delete(store.records, nonce) + + if record.Subject != subject || record.Method != method || record.Path != requestPath { + return fmt.Errorf("replay nonce does not match request context") + } + + return nil +} + +func (store *Store) extractSubject(r *http.Request) string { + if store == nil || store.subjectExtractor == nil { + return DefaultSubjectExtractor(r) + } + return store.subjectExtractor(r) +} + +func (store *Store) cleanupExpiredLocked(now time.Time) { + for nonce, record := range store.records { + if now.After(record.ExpiresAt) { + delete(store.records, nonce) + } + } +} + +func normalizeScope(method, requestPath string) (string, string, error) { + normalizedMethod := strings.ToUpper(strings.TrimSpace(method)) + switch normalizedMethod { + case http.MethodPost, http.MethodPut, http.MethodDelete: + default: + return "", "", fmt.Errorf("unsupported replay-protected method [%s]", method) + } + + normalizedPath := strings.TrimSpace(requestPath) + if normalizedPath == "" { + return "", "", fmt.Errorf("request path can not be empty") + } + if !strings.HasPrefix(normalizedPath, "/") { + normalizedPath = "/" + normalizedPath + } + normalizedPath = pathutil.Clean(normalizedPath) + if normalizedPath == "." { + normalizedPath = "/" + } + + return normalizedMethod, normalizedPath, nil +} diff --git a/core/security/replay/replay_test.go b/core/security/replay/replay_test.go new file mode 100644 index 000000000..b495b71c9 --- /dev/null +++ b/core/security/replay/replay_test.go @@ -0,0 +1,59 @@ +package replay + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestReplayNonceCanOnlyBeUsedOnce(t *testing.T) { + store := NewStore(StoreOptions{}) + req := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) + + nonce, _, err := store.IssueReplayNonce(req, http.MethodPost, "/account/login") + if err != nil { + t.Fatalf("issue replay nonce failed: %v", err) + } + + req.Header.Set(HeaderName, nonce) + if err := store.ValidateAndConsumeReplayNonce(req); err != nil { + t.Fatalf("expected first nonce use to succeed: %v", err) + } + if err := store.ValidateAndConsumeReplayNonce(req); err == nil { + t.Fatal("expected second nonce use to be rejected") + } +} + +func TestReplayNonceBindsToAuthorizationHeader(t *testing.T) { + store := NewStore(StoreOptions{}) + issueReq := httptest.NewRequest(http.MethodPut, "https://console.local/credential/test", nil) + issueReq.Header.Set("Authorization", "Bearer token-a") + + nonce, _, err := store.IssueReplayNonce(issueReq, http.MethodPut, "/credential/test") + if err != nil { + t.Fatalf("issue replay nonce failed: %v", err) + } + + useReq := httptest.NewRequest(http.MethodPut, "https://console.local/credential/test", nil) + useReq.Header.Set(HeaderName, nonce) + useReq.Header.Set("Authorization", "Bearer token-b") + if err := store.ValidateAndConsumeReplayNonce(useReq); err == nil { + t.Fatal("expected nonce bound to a different authorization header to fail") + } +} + +func TestReplayNonceBindsToPathAndMethod(t *testing.T) { + store := NewStore(StoreOptions{}) + issueReq := httptest.NewRequest(http.MethodPost, "https://console.local/setup/_initialize", nil) + + nonce, _, err := store.IssueReplayNonce(issueReq, http.MethodPost, "/setup/_initialize") + if err != nil { + t.Fatalf("issue replay nonce failed: %v", err) + } + + useReq := httptest.NewRequest(http.MethodPut, "https://console.local/setup/_initialize", nil) + useReq.Header.Set(HeaderName, nonce) + if err := store.ValidateAndConsumeReplayNonce(useReq); err == nil { + t.Fatal("expected nonce with mismatched method to fail") + } +} From 13c81cfb436342d7eb8f71e5af3a825fd71f34ad Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 23 May 2026 08:29:03 +0800 Subject: [PATCH 045/137] improve: refactor for the code --- core/api/websocket/reverse/manager.go | 312 ++++++++++++++++++++ core/api/websocket/reverse/manager_test.go | 73 +++++ core/api/websocket/reverse/protocol.go | 162 ++++++++++ core/api/websocket/reverse/protocol_test.go | 43 +++ 4 files changed, 590 insertions(+) create mode 100644 core/api/websocket/reverse/manager.go create mode 100644 core/api/websocket/reverse/manager_test.go create mode 100644 core/api/websocket/reverse/protocol.go create mode 100644 core/api/websocket/reverse/protocol_test.go diff --git a/core/api/websocket/reverse/manager.go b/core/api/websocket/reverse/manager.go new file mode 100644 index 000000000..2a040e1e4 --- /dev/null +++ b/core/api/websocket/reverse/manager.go @@ -0,0 +1,312 @@ +package reverse + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "strings" + "sync" + "time" + + "infini.sh/framework/core/util" +) + +const ( + DefaultTimeout = 30 * time.Second + DefaultMaxResponseBytes = 8 * 1024 * 1024 + DefaultReconnectWait = 6 * time.Second + DefaultReconnectPoll = 200 * time.Millisecond +) + +var ( + ErrDisconnected = errors.New("reverse channel disconnected") + ErrNotConnected = errors.New("reverse channel is not connected") +) + +type ManagerOptions struct { + DefaultTimeout time.Duration + MaxResponseBytes int + ReconnectWait time.Duration + ReconnectPoll time.Duration +} + +type pendingResponse struct { + peerID string + body bytes.Buffer + status int + err error + done chan struct{} + completed bool +} + +type SessionManager struct { + options ManagerOptions + mu sync.Mutex + pendingSessions map[string]string + activeSessions map[string]string + activeSessionsByID map[string]string + pendingResponses map[string]*pendingResponse +} + +func NewSessionManager(options ManagerOptions) *SessionManager { + if options.DefaultTimeout <= 0 { + options.DefaultTimeout = DefaultTimeout + } + if options.MaxResponseBytes <= 0 { + options.MaxResponseBytes = DefaultMaxResponseBytes + } + if options.ReconnectWait <= 0 { + options.ReconnectWait = DefaultReconnectWait + } + if options.ReconnectPoll <= 0 { + options.ReconnectPoll = DefaultReconnectPoll + } + return &SessionManager{ + options: options, + pendingSessions: map[string]string{}, + activeSessions: map[string]string{}, + activeSessionsByID: map[string]string{}, + pendingResponses: map[string]*pendingResponse{}, + } +} + +func (m *SessionManager) RegisterPendingSession(sessionID, peerID string) { + m.mu.Lock() + defer m.mu.Unlock() + m.pendingSessions[sessionID] = strings.TrimSpace(peerID) +} + +func (m *SessionManager) ActivateSession(sessionID, peerID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + peerID = strings.TrimSpace(peerID) + if expectedPeerID, ok := m.pendingSessions[sessionID]; !ok || expectedPeerID != peerID { + return fmt.Errorf("session handshake mismatch") + } + delete(m.pendingSessions, sessionID) + + if previousSession, ok := m.activeSessions[peerID]; ok && previousSession != sessionID { + delete(m.activeSessionsByID, previousSession) + } + + m.activeSessions[peerID] = sessionID + m.activeSessionsByID[sessionID] = peerID + return nil +} + +func (m *SessionManager) HandleHelloPayload(payload string) error { + msg, err := ParseHelloPayload(payload) + if err != nil { + return err + } + return m.ActivateSession(msg.SessionID, msg.PeerID) +} + +func (m *SessionManager) HandleResponsePayload(payload string) error { + msg, err := ParseResponsePayload(payload) + if err != nil { + return err + } + m.acceptResponse(msg) + return nil +} + +func (m *SessionManager) OnDisconnect(sessionID string) { + m.mu.Lock() + defer m.mu.Unlock() + + delete(m.pendingSessions, sessionID) + peerID, ok := m.activeSessionsByID[sessionID] + if !ok { + return + } + + delete(m.activeSessionsByID, sessionID) + if currentSession, exists := m.activeSessions[peerID]; exists && currentSession == sessionID { + delete(m.activeSessions, peerID) + } + m.failPendingLocked(peerID, ErrDisconnected) +} + +func (m *SessionManager) IsConnected(peerID string) bool { + m.mu.Lock() + defer m.mu.Unlock() + sessionID, ok := m.activeSessions[peerID] + return ok && sessionID != "" +} + +func (m *SessionManager) WaitForReconnect(ctx context.Context, peerID string) bool { + waitCtx, cancel := context.WithTimeout(ctx, m.options.ReconnectWait) + defer cancel() + + if m.IsConnected(peerID) { + return true + } + + ticker := time.NewTicker(m.options.ReconnectPoll) + defer ticker.Stop() + + for { + select { + case <-waitCtx.Done(): + return false + case <-ticker.C: + if m.IsConnected(peerID) { + return true + } + } + } +} + +func IsRecoverableError(err error) bool { + return errors.Is(err, ErrDisconnected) || errors.Is(err, ErrNotConnected) +} + +func (m *SessionManager) ProxyRequest(peerID string, req *util.Request, headers http.Header, send func(sessionID, payload string) error, responseObjectToUnmarshal interface{}) (*util.Result, error) { + if req == nil { + return nil, fmt.Errorf("request is nil") + } + + ctx := req.Context + if ctx == nil { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(context.Background(), m.options.DefaultTimeout) + defer cancel() + } else if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, m.options.DefaultTimeout) + defer cancel() + } + + var lastErr error + for attempt := 0; attempt < 2; attempt++ { + res, err := m.proxyRequestOnce(ctx, strings.TrimSpace(peerID), req, headers, send, responseObjectToUnmarshal) + if err == nil { + return res, nil + } + lastErr = err + if attempt == 0 && IsRecoverableError(err) && m.WaitForReconnect(ctx, peerID) { + continue + } + return res, err + } + return nil, lastErr +} + +func (m *SessionManager) proxyRequestOnce(ctx context.Context, peerID string, req *util.Request, headers http.Header, send func(sessionID, payload string) error, responseObjectToUnmarshal interface{}) (*util.Result, error) { + requestID := util.GetUUID() + msg := RequestMessage{ + RequestID: requestID, + PeerID: peerID, + Method: req.Method, + Path: req.Path, + Headers: headers, + } + msg.SetBody(req.Body) + if authorization := strings.TrimSpace(msg.Headers.Get("Authorization")); strings.HasPrefix(strings.ToLower(authorization), "bearer ") { + msg.AccessToken = strings.TrimSpace(authorization[7:]) + } + + pending := &pendingResponse{ + peerID: peerID, + done: make(chan struct{}), + } + + m.mu.Lock() + sessionID, ok := m.activeSessions[peerID] + if !ok || sessionID == "" { + m.mu.Unlock() + return nil, fmt.Errorf("%w for peer [%s]", ErrNotConnected, peerID) + } + m.pendingResponses[requestID] = pending + m.mu.Unlock() + + if err := send(sessionID, FormatRequestCommand(msg)); err != nil { + m.mu.Lock() + delete(m.pendingResponses, requestID) + m.mu.Unlock() + return nil, err + } + + select { + case <-pending.done: + case <-ctx.Done(): + m.mu.Lock() + delete(m.pendingResponses, requestID) + m.mu.Unlock() + return nil, ctx.Err() + } + + if pending.err != nil { + return nil, pending.err + } + + res := &util.Result{ + Body: pending.body.Bytes(), + StatusCode: pending.status, + } + if res.StatusCode != http.StatusOK { + return res, fmt.Errorf("request error: %s", string(res.Body)) + } + if responseObjectToUnmarshal != nil && len(res.Body) > 0 { + return res, util.FromJSONBytes(res.Body, responseObjectToUnmarshal) + } + return res, nil +} + +func (m *SessionManager) acceptResponse(msg ResponseMessage) { + m.mu.Lock() + defer m.mu.Unlock() + + pending, ok := m.pendingResponses[msg.RequestID] + if !ok || pending.completed { + return + } + if msg.PeerID != "" && pending.peerID != "" && msg.PeerID != pending.peerID { + return + } + + if msg.Chunk != "" { + chunk, err := msg.ChunkBytes() + if err != nil { + m.completePendingLocked(msg.RequestID, pending, 0, fmt.Errorf("decode reverse response chunk: %w", err)) + return + } + if pending.body.Len()+len(chunk) > m.options.MaxResponseBytes { + m.completePendingLocked(msg.RequestID, pending, 0, fmt.Errorf("reverse response exceeds %d bytes", m.options.MaxResponseBytes)) + return + } + _, _ = pending.body.Write(chunk) + } + + if msg.Done { + status := msg.Status + if status == 0 { + status = http.StatusOK + } + m.completePendingLocked(msg.RequestID, pending, status, nil) + } +} + +func (m *SessionManager) completePendingLocked(requestID string, pending *pendingResponse, status int, err error) { + if pending.completed { + return + } + pending.completed = true + pending.status = status + pending.err = err + close(pending.done) + delete(m.pendingResponses, requestID) +} + +func (m *SessionManager) failPendingLocked(peerID string, err error) { + for requestID, pending := range m.pendingResponses { + if pending.peerID != peerID { + continue + } + m.completePendingLocked(requestID, pending, 0, err) + } +} diff --git a/core/api/websocket/reverse/manager_test.go b/core/api/websocket/reverse/manager_test.go new file mode 100644 index 000000000..bc94cb97b --- /dev/null +++ b/core/api/websocket/reverse/manager_test.go @@ -0,0 +1,73 @@ +package reverse + +import ( + "net/http" + "strings" + "testing" + + "infini.sh/framework/core/util" +) + +func TestSessionManagerProxyRequestRoundTrip(t *testing.T) { + manager := NewSessionManager(ManagerOptions{}) + manager.RegisterPendingSession("session-1", "peer-1") + if err := manager.ActivateSession("session-1", "peer-1"); err != nil { + t.Fatalf("activate session: %v", err) + } + + headers := http.Header{} + headers.Set("Authorization", "Bearer token-1") + + send := func(sessionID, payload string) error { + if sessionID != "session-1" { + t.Fatalf("unexpected session id: %s", sessionID) + } + if !strings.HasPrefix(payload, RequestCommand+" ") { + t.Fatalf("unexpected payload: %s", payload) + } + msg, err := ParseRequestPayload(strings.TrimPrefix(payload, RequestCommand+" ")) + if err != nil { + t.Fatalf("parse request payload: %v", err) + } + if msg.BearerToken() != "token-1" { + t.Fatalf("unexpected bearer token: %s", msg.BearerToken()) + } + return WriteChunkedResponse(func(responsePayload string) error { + if !strings.HasPrefix(responsePayload, ResponseCommand+" ") { + t.Fatalf("unexpected response payload: %s", responsePayload) + } + return manager.HandleResponsePayload(strings.TrimPrefix(responsePayload, ResponseCommand+" ")) + }, msg.RequestID, msg.PeerID, http.StatusOK, []byte(`{"ack":true}`), DefaultResponseChunkBytes) + } + + var response map[string]bool + req := &util.Request{Method: http.MethodGet, Path: "/stats"} + res, err := manager.ProxyRequest("peer-1", req, headers, send, &response) + if err != nil { + t.Fatalf("proxy request: %v", err) + } + if res.StatusCode != http.StatusOK { + t.Fatalf("unexpected status: %d", res.StatusCode) + } + if !response["ack"] { + t.Fatal("expected response to unmarshal") + } +} + +func TestSessionManagerDisconnectFailsPendingRequest(t *testing.T) { + manager := NewSessionManager(ManagerOptions{}) + manager.RegisterPendingSession("session-1", "peer-1") + if err := manager.ActivateSession("session-1", "peer-1"); err != nil { + t.Fatalf("activate session: %v", err) + } + + send := func(sessionID, payload string) error { + manager.OnDisconnect(sessionID) + return nil + } + + _, err := manager.ProxyRequest("peer-1", &util.Request{Method: http.MethodGet, Path: "/stats"}, nil, send, nil) + if !IsRecoverableError(err) { + t.Fatalf("expected recoverable disconnect error, got %v", err) + } +} diff --git a/core/api/websocket/reverse/protocol.go b/core/api/websocket/reverse/protocol.go new file mode 100644 index 000000000..173337c88 --- /dev/null +++ b/core/api/websocket/reverse/protocol.go @@ -0,0 +1,162 @@ +package reverse + +import ( + "encoding/base64" + "net/http" + "strings" + + "infini.sh/framework/core/util" +) + +const ( + HeaderPeerID = "X-INFINI-INSTANCE-ID" + HelloCommand = "reverse_hello" + RequestCommand = "reverse_request" + ResponseCommand = "reverse_response" + DefaultResponseChunkBytes = 32 * 1024 +) + +type HelloMessage struct { + SessionID string `json:"session_id"` + PeerID string `json:"instance_id"` +} + +type RequestMessage struct { + RequestID string `json:"request_id"` + PeerID string `json:"instance_id"` + Method string `json:"method"` + Path string `json:"path"` + Body string `json:"body,omitempty"` + Headers http.Header `json:"headers,omitempty"` + AccessToken string `json:"access_token,omitempty"` +} + +type ResponseMessage struct { + RequestID string `json:"request_id"` + PeerID string `json:"instance_id"` + Chunk string `json:"chunk,omitempty"` + Status int `json:"status,omitempty"` + Done bool `json:"done,omitempty"` +} + +func ParseHelloPayload(payload string) (HelloMessage, error) { + msg := HelloMessage{} + return msg, util.FromJSONBytes([]byte(payload), &msg) +} + +func ParseRequestPayload(payload string) (RequestMessage, error) { + msg := RequestMessage{} + return msg, util.FromJSONBytes([]byte(payload), &msg) +} + +func ParseResponsePayload(payload string) (ResponseMessage, error) { + msg := ResponseMessage{} + return msg, util.FromJSONBytes([]byte(payload), &msg) +} + +func FormatHelloCommand(msg HelloMessage) string { + return HelloCommand + " " + string(util.MustToJSONBytes(msg)) +} + +func FormatRequestCommand(msg RequestMessage) string { + return RequestCommand + " " + string(util.MustToJSONBytes(msg)) +} + +func FormatResponseCommand(msg ResponseMessage) string { + return ResponseCommand + " " + string(util.MustToJSONBytes(msg)) +} + +func (m *RequestMessage) SetBody(body []byte) { + if len(body) == 0 { + m.Body = "" + return + } + m.Body = base64.StdEncoding.EncodeToString(body) +} + +func (m RequestMessage) BodyBytes() ([]byte, error) { + if m.Body == "" { + return nil, nil + } + return base64.StdEncoding.DecodeString(m.Body) +} + +func (m RequestMessage) NormalizedHeaders() http.Header { + headers := http.Header{} + for key, values := range m.Headers { + copied := append([]string(nil), values...) + headers[key] = copied + } + if headers.Get("Authorization") == "" && strings.TrimSpace(m.AccessToken) != "" { + headers.Set("Authorization", "Bearer "+strings.TrimSpace(m.AccessToken)) + } + return headers +} + +func (m RequestMessage) ApplyHeaders(req *http.Request) { + if req == nil { + return + } + if req.Header == nil { + req.Header = http.Header{} + } + for key := range req.Header { + req.Header.Del(key) + } + for key, values := range m.NormalizedHeaders() { + for _, value := range values { + req.Header.Add(key, value) + } + } +} + +func (m RequestMessage) BearerToken() string { + value := strings.TrimSpace(m.NormalizedHeaders().Get("Authorization")) + if !strings.HasPrefix(strings.ToLower(value), "bearer ") { + return "" + } + return strings.TrimSpace(value[7:]) +} + +func (m *ResponseMessage) SetChunk(body []byte) { + if len(body) == 0 { + m.Chunk = "" + return + } + m.Chunk = base64.StdEncoding.EncodeToString(body) +} + +func (m ResponseMessage) ChunkBytes() ([]byte, error) { + if m.Chunk == "" { + return nil, nil + } + return base64.StdEncoding.DecodeString(m.Chunk) +} + +func WriteChunkedResponse(write func(payload string) error, requestID, peerID string, status int, body []byte, chunkBytes int) error { + if chunkBytes <= 0 { + chunkBytes = DefaultResponseChunkBytes + } + for start := 0; start < len(body); start += chunkBytes { + end := start + chunkBytes + if end > len(body) { + end = len(body) + } + msg := ResponseMessage{ + RequestID: requestID, + PeerID: peerID, + } + msg.SetChunk(body[start:end]) + if err := write(FormatResponseCommand(msg)); err != nil { + return err + } + } + + done := ResponseMessage{ + RequestID: requestID, + PeerID: peerID, + Status: status, + Done: true, + } + return write(FormatResponseCommand(done)) +} diff --git a/core/api/websocket/reverse/protocol_test.go b/core/api/websocket/reverse/protocol_test.go new file mode 100644 index 000000000..a2085d9da --- /dev/null +++ b/core/api/websocket/reverse/protocol_test.go @@ -0,0 +1,43 @@ +package reverse + +import ( + "net/http" + "testing" +) + +func TestRequestMessageNormalizedHeadersFallsBackToLegacyAccessToken(t *testing.T) { + msg := RequestMessage{ + AccessToken: "token-1", + } + + headers := msg.NormalizedHeaders() + if got := headers.Get("Authorization"); got != "Bearer token-1" { + t.Fatalf("unexpected authorization header: %s", got) + } + if got := msg.BearerToken(); got != "token-1" { + t.Fatalf("unexpected bearer token: %s", got) + } +} + +func TestRequestMessageApplyHeaders(t *testing.T) { + msg := RequestMessage{ + Headers: http.Header{ + "Authorization": []string{"Bearer token-2"}, + "X-Test": []string{"value"}, + }, + } + req, _ := http.NewRequest(http.MethodGet, "http://example.com", nil) + req.Header.Set("Existing", "old") + + msg.ApplyHeaders(req) + + if req.Header.Get("Existing") != "" { + t.Fatal("expected old header to be removed") + } + if req.Header.Get("Authorization") != "Bearer token-2" { + t.Fatalf("unexpected authorization header: %s", req.Header.Get("Authorization")) + } + if req.Header.Get("X-Test") != "value" { + t.Fatalf("unexpected x-test header: %s", req.Header.Get("X-Test")) + } +} From 10cfb4f243f67eed8446b376cc2c2a890c263f1c Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 23 May 2026 09:02:37 +0800 Subject: [PATCH 046/137] improve: refactor for the code --- core/model/instance.go | 18 ++++++++++++++++++ core/model/instance_test.go | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/core/model/instance.go b/core/model/instance.go index 6b9e1abfe..c090bc5c4 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -140,6 +140,23 @@ func resolveManagedInstanceEndpoint(apiConfig config.APIConfig, webConfig config return apiConfig.GetEndpoint() } +func buildManagedInstanceServices(apiConfig config.APIConfig, webConfig config.WebAppConfig) []ServiceInfo { + services := []ServiceInfo{} + if apiConfig.Enabled { + services = append(services, ServiceInfo{ + Name: "api", + Endpoint: apiConfig.GetEndpoint(), + }) + } + if webConfig.Enabled { + services = append(services, ServiceInfo{ + Name: "web", + Endpoint: webConfig.GetEndpoint(), + }) + } + return services +} + func GetInstanceInfo() Instance { instance := Instance{} instance.ID = global.Env().SystemConfig.NodeConfig.ID @@ -152,6 +169,7 @@ func GetInstanceInfo() Instance { _, publicIP, _, _ := util.GetPublishNetworkDeviceInfo(global.Env().SystemConfig.NodeConfig.MajorIpPattern) instance.Endpoint = resolveManagedInstanceEndpoint(global.Env().SystemConfig.APIConfig, global.Env().SystemConfig.WebAppConfig) + instance.Services = buildManagedInstanceServices(global.Env().SystemConfig.APIConfig, global.Env().SystemConfig.WebAppConfig) ips := util.GetLocalIPs() if len(ips) > 0 { diff --git a/core/model/instance_test.go b/core/model/instance_test.go index be13ca778..9713684de 100644 --- a/core/model/instance_test.go +++ b/core/model/instance_test.go @@ -31,3 +31,21 @@ func TestResolveManagedInstanceEndpoint(t *testing.T) { } }) } + +func TestBuildManagedInstanceServices(t *testing.T) { + apiConfig := config.APIConfig{Enabled: true} + apiConfig.NetworkConfig.Publish = "127.0.0.1:2900" + webConfig := config.WebAppConfig{Enabled: true} + webConfig.NetworkConfig.Publish = "127.0.0.1:8080" + + services := buildManagedInstanceServices(apiConfig, webConfig) + if len(services) != 2 { + t.Fatalf("unexpected service count: %#v", services) + } + if services[0].Name != "api" || services[0].Endpoint != "http://127.0.0.1:2900" { + t.Fatalf("unexpected api service: %#v", services[0]) + } + if services[1].Name != "web" || services[1].Endpoint != "http://127.0.0.1:8080" { + t.Fatalf("unexpected web service: %#v", services[1]) + } +} From 38b79a19fb4a7790ed97b80a98619ee9864a936a Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 23 May 2026 10:59:32 +0800 Subject: [PATCH 047/137] improve: agent reverse channel endpoint use array --- modules/configs/common/config.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/configs/common/config.go b/modules/configs/common/config.go index ea622b240..a13c8675e 100644 --- a/modules/configs/common/config.go +++ b/modules/configs/common/config.go @@ -38,12 +38,12 @@ type AgentConfig struct { } type SetupConfig struct { - DownloadURL string `config:"download_url"` - CACertFile string `config:"ca_cert"` - CAKeyFile string `config:"ca_key"` - ConsoleEndpoint string `config:"console_endpoint"` - ReverseChannelEndpoint string `config:"reverse_channel_endpoint"` - Port string `config:"port"` + DownloadURL string `config:"download_url"` + CACertFile string `config:"ca_cert"` + CAKeyFile string `config:"ca_key"` + ConsoleEndpoint string `config:"console_endpoint"` + ReverseChannelEndpoints []string `config:"reverse_channel_endpoints"` + Port string `config:"port"` } func GetAgentConfig() *AgentConfig { From 513cc7df042bf51a8cd61d6c0e920afb91a90680 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 23 May 2026 18:38:35 +0800 Subject: [PATCH 048/137] fix(api): avoid duplicate embedded websocket routes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- core/api/web.go | 22 +++++++++++++++++- core/api/web_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 core/api/web_test.go diff --git a/core/api/web.go b/core/api/web.go index c4617aff8..c50504dc5 100755 --- a/core/api/web.go +++ b/core/api/web.go @@ -134,7 +134,10 @@ func StartWeb(cfg config.WebAppConfig) { if cfg.WebsocketConfig.Enabled { websocket.InitWebSocket(cfg.WebsocketConfig) - uiServeMux.HandleFunc("/ws", websocket.ServeWs) + websocketPath := getWebsocketRegistrationPath(cfg) + if shouldRegisterWebsocketOnWeb(cfg) { + uiServeMux.HandleFunc(websocketPath, websocket.ServeWs) + } if registeredWebSocketCommandHandler != nil { for k, v := range registeredWebSocketCommandHandler { log.Debug("register websocket handler: ", k, " ", v) @@ -315,6 +318,23 @@ func (i *InterceptorHandler) AddInterceptors(interceptors ...Interceptor) { } } +func getWebsocketRegistrationPath(cfg config.WebAppConfig) string { + if cfg.WebsocketConfig.BasePath != "" { + return cfg.WebsocketConfig.BasePath + } + return "/ws" +} + +func shouldRegisterWebsocketOnWeb(cfg config.WebAppConfig) bool { + if !cfg.WebsocketConfig.Enabled { + return false + } + if !cfg.EmbeddingAPI || registeredAPIFuncHandler == nil { + return true + } + return registeredAPIFuncHandler[getWebsocketRegistrationPath(cfg)] == nil +} + func (i *InterceptorHandler) Handler(handler http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { var appliedInterceptors []Interceptor diff --git a/core/api/web_test.go b/core/api/web_test.go new file mode 100644 index 000000000..0bdc5a7a9 --- /dev/null +++ b/core/api/web_test.go @@ -0,0 +1,53 @@ +package api + +import ( + "net/http" + "testing" + + "infini.sh/framework/core/config" +) + +func TestWebsocketRegistrationPath(t *testing.T) { + cfg := config.WebAppConfig{} + cfg.WebsocketConfig.Enabled = true + cfg.WebsocketConfig.BasePath = "/custom-ws" + + if got := getWebsocketRegistrationPath(cfg); got != "/custom-ws" { + t.Fatalf("unexpected websocket path: %s", got) + } + + cfg.WebsocketConfig.BasePath = "" + if got := getWebsocketRegistrationPath(cfg); got != "/ws" { + t.Fatalf("unexpected default websocket path: %s", got) + } +} + +func TestShouldRegisterWebsocketOnWeb(t *testing.T) { + originalHandlers := registeredAPIFuncHandler + t.Cleanup(func() { + registeredAPIFuncHandler = originalHandlers + }) + + cfg := config.WebAppConfig{} + cfg.WebsocketConfig.Enabled = true + cfg.WebsocketConfig.BasePath = "/ws" + cfg.EmbeddingAPI = true + + registeredAPIFuncHandler = map[string]func(http.ResponseWriter, *http.Request){ + "/ws": func(http.ResponseWriter, *http.Request) {}, + } + + if shouldRegisterWebsocketOnWeb(cfg) { + t.Fatal("expected embedded API websocket registration to suppress duplicate web registration") + } + + delete(registeredAPIFuncHandler, "/ws") + if !shouldRegisterWebsocketOnWeb(cfg) { + t.Fatal("expected websocket registration when no embedded API websocket handler exists") + } + + cfg.EmbeddingAPI = false + if !shouldRegisterWebsocketOnWeb(cfg) { + t.Fatal("expected websocket registration when embedding_api is disabled") + } +} From 8337c84eb5cd6e7270b9ab5c170e0d461b183408 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 23 May 2026 19:22:21 +0800 Subject: [PATCH 049/137] fix(api): keep embedded API off UI root Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- core/api/web.go | 14 ++++++++++++++ core/api/web_test.go | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/core/api/web.go b/core/api/web.go index c50504dc5..cd72cd121 100755 --- a/core/api/web.go +++ b/core/api/web.go @@ -119,6 +119,9 @@ func StartWeb(cfg config.WebAppConfig) { if registeredAPIMethodHandler != nil { for k, v := range registeredAPIMethodHandler { for m, n := range v { + if shouldSkipEmbeddedAPIRoute(m) { + continue + } log.Debug("register http handler: ", k, " ", m) uiRouter.Handle(k, m, n) } @@ -126,6 +129,9 @@ func StartWeb(cfg config.WebAppConfig) { } if registeredAPIFuncHandler != nil { for k, v := range registeredAPIFuncHandler { + if shouldSkipEmbeddedAPIRoute(k) { + continue + } log.Debug("register http handler: ", k) uiServeMux.HandleFunc(k, v) } @@ -335,6 +341,14 @@ func shouldRegisterWebsocketOnWeb(cfg config.WebAppConfig) bool { return registeredAPIFuncHandler[getWebsocketRegistrationPath(cfg)] == nil } +func shouldSkipEmbeddedAPIRoute(path string) bool { + if registeredUIHandler == nil { + return false + } + _, exists := registeredUIHandler[path] + return exists +} + func (i *InterceptorHandler) Handler(handler http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { var appliedInterceptors []Interceptor diff --git a/core/api/web_test.go b/core/api/web_test.go index 0bdc5a7a9..01e0949d9 100644 --- a/core/api/web_test.go +++ b/core/api/web_test.go @@ -51,3 +51,21 @@ func TestShouldRegisterWebsocketOnWeb(t *testing.T) { t.Fatal("expected websocket registration when embedding_api is disabled") } } + +func TestShouldSkipEmbeddedAPIRoute(t *testing.T) { + originalUIHandlers := registeredUIHandler + t.Cleanup(func() { + registeredUIHandler = originalUIHandlers + }) + + registeredUIHandler = map[string]http.Handler{ + "/": http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), + } + + if !shouldSkipEmbeddedAPIRoute("/") { + t.Fatal("expected API root route to be skipped when UI root is registered") + } + if shouldSkipEmbeddedAPIRoute("/_info") { + t.Fatal("expected unrelated API route not to be skipped") + } +} From 3d2145b429d254979ebd2be74adf095a1f31f724 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 24 May 2026 05:50:25 +0800 Subject: [PATCH 050/137] fix(configs): inherit manager tls for config clients Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- core/env/http_client.go | 24 ++++++++++++++ core/env/http_client_test.go | 63 ++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 core/env/http_client_test.go diff --git a/core/env/http_client.go b/core/env/http_client.go index f5ba79253..e154e2ac5 100644 --- a/core/env/http_client.go +++ b/core/env/http_client.go @@ -41,6 +41,30 @@ func (env *Env) GetHTTPClientConfig(name, endpoint string) *config.HTTPClientCon TLSConfig: config.TLSConfig{SkipDomainVerify: true, TLSInsecureSkipVerify: true}, } } + if name == "configs" && ((!ok && !isZeroTLSConfig(env.SystemConfig.Configs.TLSConfig)) || isZeroTLSConfig(clientCfg.TLSConfig)) { + clientCfg.TLSConfig = env.SystemConfig.Configs.TLSConfig + } //TODO support client config per endpoint return &clientCfg } + +func isZeroTLSConfig(cfg config.TLSConfig) bool { + return !cfg.TLSEnabled && + cfg.TLSCertFile == "" && + cfg.TLSCertPassword == "" && + cfg.TLSKeyFile == "" && + cfg.TLSCACertFile == "" && + !cfg.TLSInsecureSkipVerify && + cfg.DefaultDomain == "" && + !cfg.SkipDomainVerify && + cfg.ClientSessionCacheSize == 0 && + !cfg.TLSBypassMalformedCert && + !cfg.AutoIssue.Enabled && + cfg.AutoIssue.Email == "" && + cfg.AutoIssue.Path == "" && + !cfg.AutoIssue.IncludeDefaultDomain && + !cfg.AutoIssue.SkipInvalidDomain && + len(cfg.AutoIssue.Domains) == 0 && + cfg.AutoIssue.Provider.TencentDNS.SecretID == "" && + cfg.AutoIssue.Provider.TencentDNS.SecretKey == "" +} diff --git a/core/env/http_client_test.go b/core/env/http_client_test.go new file mode 100644 index 000000000..1fb96ec11 --- /dev/null +++ b/core/env/http_client_test.go @@ -0,0 +1,63 @@ +package env + +import ( + "testing" + + "infini.sh/framework/core/config" +) + +func TestGetHTTPClientConfigFallsBackToConfigsTLS(t *testing.T) { + env := &Env{ + SystemConfig: &config.SystemConfig{ + Configs: config.ConfigsConfig{ + TLSConfig: config.TLSConfig{ + TLSEnabled: true, + TLSCertFile: "config/client.crt", + TLSKeyFile: "config/client.key", + TLSCACertFile: "config/ca.crt", + TLSInsecureSkipVerify: false, + SkipDomainVerify: true, + ClientSessionCacheSize: 64, + }, + }, + }, + } + + cfg := env.GetHTTPClientConfig("configs", "") + if cfg.TLSConfig.TLSCertFile != "config/client.crt" { + t.Fatalf("expected configs tls cert_file fallback, got %q", cfg.TLSConfig.TLSCertFile) + } + if cfg.TLSConfig.TLSKeyFile != "config/client.key" { + t.Fatalf("expected configs tls key_file fallback, got %q", cfg.TLSConfig.TLSKeyFile) + } + if cfg.TLSConfig.TLSCACertFile != "config/ca.crt" { + t.Fatalf("expected configs tls ca_file fallback, got %q", cfg.TLSConfig.TLSCACertFile) + } + if !cfg.TLSConfig.SkipDomainVerify { + t.Fatal("expected configs tls skip_domain_verify fallback") + } +} + +func TestGetHTTPClientConfigKeepsExplicitConfigsClientTLS(t *testing.T) { + env := &Env{ + SystemConfig: &config.SystemConfig{ + Configs: config.ConfigsConfig{ + TLSConfig: config.TLSConfig{ + TLSCertFile: "config/client.crt", + }, + }, + HTTPClientConfig: map[string]config.HTTPClientConfig{ + "configs": { + TLSConfig: config.TLSConfig{ + TLSCertFile: "override/client.crt", + }, + }, + }, + }, + } + + cfg := env.GetHTTPClientConfig("configs", "") + if cfg.TLSConfig.TLSCertFile != "override/client.crt" { + t.Fatalf("expected explicit configs client tls to win, got %q", cfg.TLSConfig.TLSCertFile) + } +} From c4f398a09dfabda9d1822e018b744ed0bc7ee9c4 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 26 May 2026 16:19:05 +0800 Subject: [PATCH 051/137] feat: sync native challenge login into console_framework\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- core/api/security.go | 55 +++- core/api/security_test.go | 54 +++ core/security/password_challenge.go | 132 ++++++++ core/security/password_challenge_test.go | 108 ++++++ .../passwordchallenge/password_challenge.go | 32 +- .../password_challenge_test.go | 52 ++- core/security/replay/replay.go | 14 + core/security/replay/replay_test.go | 69 ++++ core/security/session.go | 7 +- core/security/user_profile.go | 10 +- core/security/user_session.go | 5 +- core/security/validate.go | 2 +- docs/content.en/docs/release-notes/_index.md | 10 +- go.mod | 1 + go.sum | 2 + modules/security/account/profile.go | 13 +- modules/security/http_filters/json_mask.go | 12 +- modules/security/http_filters/security.go | 75 +++++ .../security/http_filters/security_test.go | 128 ++++++++ modules/security/rbac/account_login.go | 307 ++++++++++++++++++ modules/security/rbac/account_login_test.go | 190 +++++++++++ modules/security/rbac/init.go | 1 + modules/security/rbac/user.go | 28 +- 23 files changed, 1267 insertions(+), 40 deletions(-) create mode 100644 core/security/password_challenge.go create mode 100644 core/security/password_challenge_test.go create mode 100644 modules/security/http_filters/security.go create mode 100644 modules/security/http_filters/security_test.go create mode 100644 modules/security/rbac/account_login.go create mode 100644 modules/security/rbac/account_login_test.go diff --git a/core/api/security.go b/core/api/security.go index 67226a018..89a74cd48 100644 --- a/core/api/security.go +++ b/core/api/security.go @@ -1,3 +1,26 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package api import ( @@ -9,9 +32,21 @@ import ( ) type SecureTransportOptions struct { + // TrustForwardHeaders allows HTTPS detection to honor reverse-proxy forwarding headers. TrustForwardHeaders bool } +const ( + // FeatureRequireSecureTransport marks a UI handler as HTTPS-only when it is enforced by filters. + FeatureRequireSecureTransport = "feature_require_secure_transport" + // FeatureRequireReplayProtection marks a UI handler as requiring a valid replay nonce. + FeatureRequireReplayProtection = "feature_require_replay_protection" + // LabelTrustForwardHeaders stores whether HTTPS checks may trust reverse-proxy forwarding headers. + LabelTrustForwardHeaders = "label_trust_forward_headers" +) + +// RequestUsesSecureTransport reports whether the request arrived over HTTPS directly or, when +// allowed, through a trusted reverse proxy that forwarded HTTPS metadata. func RequestUsesSecureTransport(req *http.Request, options ...SecureTransportOptions) bool { if req == nil { return false @@ -38,21 +73,24 @@ func RequestUsesSecureTransport(req *http.Request, options ...SecureTransportOpt return forwardedHeaderIndicatesHTTPS(req.Header.Get("Forwarded")) } +// RequireSecureTransport wraps a handler so it rejects requests that do not resolve to HTTPS. func (handler Handler) RequireSecureTransport(h httprouter.Handle, options ...SecureTransportOptions) httprouter.Handle { resolved := resolveSecureTransportOptions(options) return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if !RequestUsesSecureTransport(r, resolved) { - handler.WriteError(w, "sensitive endpoints require HTTPS or a trusted HTTPS reverse proxy", http.StatusUpgradeRequired) + handler.WriteError(w, "this endpoint requires HTTPS. use https:// directly or route through a trusted HTTPS reverse proxy", http.StatusUpgradeRequired) return } h(w, r, ps) } } +// RequireSecureTransport wraps a handler with the default security handler implementation. func RequireSecureTransport(h httprouter.Handle, options ...SecureTransportOptions) httprouter.Handle { return Handler{}.RequireSecureTransport(h, options...) } +// RequireReplayProtection wraps a handler so each request must present a valid replay nonce. func (handler Handler) RequireReplayProtection(h httprouter.Handle) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { if err := replaysecurity.ValidateAndConsumeReplayNonce(r); err != nil { @@ -63,10 +101,25 @@ func (handler Handler) RequireReplayProtection(h httprouter.Handle) httprouter.H } } +// RequireReplayProtection wraps a handler with the default replay-protection implementation. func RequireReplayProtection(h httprouter.Handle) httprouter.Handle { return Handler{}.RequireReplayProtection(h) } +// SecureTransportOption annotates a UI route so SecurityFilter can enforce HTTPS consistently. +func SecureTransportOption(options ...SecureTransportOptions) Option { + resolved := resolveSecureTransportOptions(options) + return func(o *HandlerOptions) { + Feature(FeatureRequireSecureTransport)(o) + Label(LabelTrustForwardHeaders, resolved.TrustForwardHeaders)(o) + } +} + +// ReplayProtectionOption annotates a UI route so SecurityFilter enforces replay-nonce validation. +func ReplayProtectionOption() Option { + return Feature(FeatureRequireReplayProtection) +} + func resolveSecureTransportOptions(options []SecureTransportOptions) SecureTransportOptions { if len(options) == 0 { return SecureTransportOptions{} diff --git a/core/api/security_test.go b/core/api/security_test.go index 8b688b63a..ce9002e09 100644 --- a/core/api/security_test.go +++ b/core/api/security_test.go @@ -1,3 +1,26 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package api import ( @@ -10,6 +33,8 @@ import ( replaysecurity "infini.sh/framework/core/security/replay" ) +// The transport tests cover both direct TLS and trusted proxy headers because the +// security helpers are shared by embedded UI routes that may sit behind a proxy. func TestRequestUsesSecureTransport(t *testing.T) { tests := []struct { name string @@ -58,6 +83,7 @@ func TestRequestUsesSecureTransport(t *testing.T) { } } +// The wrapper should fail fast before running the protected handler on plain HTTP. func TestRequireSecureTransport(t *testing.T) { handler := Handler{} called := false @@ -79,6 +105,7 @@ func TestRequireSecureTransport(t *testing.T) { } } +// Replay-protected handlers should pass straight through once a matching nonce exists. func TestRequireReplayProtection(t *testing.T) { handler := Handler{} req := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) @@ -104,3 +131,30 @@ func TestRequireReplayProtection(t *testing.T) { t.Fatalf("expected status %d, got %d", http.StatusOK, resp.Code) } } + +// Route options are later consumed by SecurityFilter, so the feature flag and labels +// must both be set when secure transport enforcement is requested declaratively. +func TestSecureTransportOption(t *testing.T) { + options := &HandlerOptions{} + SecureTransportOption(SecureTransportOptions{TrustForwardHeaders: true})(options) + + if !options.Feature(FeatureRequireSecureTransport) { + t.Fatal("expected secure transport feature to be enabled") + } + if options.Labels == nil { + t.Fatal("expected labels to be initialized") + } + if v, ok := options.Labels[LabelTrustForwardHeaders].(bool); !ok || !v { + t.Fatalf("expected trust forward headers label to be true, got %#v", options.Labels[LabelTrustForwardHeaders]) + } +} + +// Replay protection uses a single feature flag because the filter reads no extra labels. +func TestReplayProtectionOption(t *testing.T) { + options := &HandlerOptions{} + ReplayProtectionOption()(options) + + if !options.Feature(FeatureRequireReplayProtection) { + t.Fatal("expected replay protection feature to be enabled") + } +} diff --git a/core/security/password_challenge.go b/core/security/password_challenge.go new file mode 100644 index 000000000..2c860a6ed --- /dev/null +++ b/core/security/password_challenge.go @@ -0,0 +1,132 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package security + +import ( + "errors" + + "golang.org/x/crypto/bcrypt" + passwordchallenge "infini.sh/framework/core/security/passwordchallenge" + "infini.sh/framework/core/util" +) + +const ( + // PasswordChallengeMethod identifies the login flow returned by the challenge endpoint. + PasswordChallengeMethod = passwordchallenge.Method + // PasswordChallengeAlgorithm describes the verifier/proof derivation algorithm for clients. + PasswordChallengeAlgorithm = passwordchallenge.Algorithm + // PasswordChallengeIterations tells clients which PBKDF2 work factor to use. + PasswordChallengeIterations = passwordchallenge.Iterations +) + +// LoginChallenge re-exports the framework challenge payload used by native account login. +type LoginChallenge = passwordchallenge.Challenge + +// CanUsePasswordChallenge reports whether a native account already has challenge credentials. +func CanUsePasswordChallenge(user *UserAccount) bool { + return user != nil && user.PasswordSalt != "" && user.PasswordVerifier != "" +} + +// SetPassword updates both the legacy bcrypt hash and the challenge verifier material. +func SetPassword(user *UserAccount, password string) error { + if user == nil { + return errors.New("user is nil") + } + + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return err + } + + // Store both the bcrypt hash for existing password checks and the derived + // verifier for challenge login so the two login modes stay in sync. + salt := util.GenerateSecureString(32) + verifier, err := DerivePasswordVerifier(password, salt) + if err != nil { + return err + } + + user.Password = string(hash) + user.PasswordSalt = salt + user.PasswordVerifier = verifier + return nil +} + +// EnsurePasswordChallenge derives challenge material for older accounts without changing the bcrypt hash. +func EnsurePasswordChallenge(user *UserAccount, password string) error { + if user == nil { + return errors.New("user is nil") + } + if CanUsePasswordChallenge(user) { + return nil + } + + // This is used as an in-place upgrade path for older native accounts that only + // have a bcrypt password hash from before challenge login was introduced. + salt := util.GenerateSecureString(32) + verifier, err := DerivePasswordVerifier(password, salt) + if err != nil { + return err + } + + user.PasswordSalt = salt + user.PasswordVerifier = verifier + return nil +} + +// VerifyPassword validates the plain password against the stored bcrypt hash. +func VerifyPassword(user *UserAccount, password string) error { + if user == nil { + return errors.New("user is nil") + } + if user.Password == "" { + return errors.New("password is not set") + } + return bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(password)) +} + +// DerivePasswordVerifier converts a password and salt into the stored challenge verifier. +func DerivePasswordVerifier(password, salt string) (string, error) { + return passwordchallenge.DeriveVerifier(password, salt) +} + +// BuildPasswordProof creates the challenge response that clients send to /account/login. +func BuildPasswordProof(verifier, subject, challengeID, nonce string) (string, error) { + return passwordchallenge.BuildProof(verifier, subject, challengeID, nonce) +} + +// VerifyPasswordProof checks whether a submitted proof matches the stored verifier. +func VerifyPasswordProof(verifier, subject, challengeID, nonce, proof string) bool { + return passwordchallenge.VerifyProof(verifier, subject, challengeID, nonce, proof) +} + +// NewLoginChallenge allocates a one-time challenge bound to the requested login subject. +func NewLoginChallenge(subject string) LoginChallenge { + return passwordchallenge.New(subject) +} + +// ConsumeLoginChallenge validates and removes a one-time challenge after it is used. +func ConsumeLoginChallenge(challengeID, subject string) (LoginChallenge, error) { + return passwordchallenge.Consume(challengeID, subject) +} diff --git a/core/security/password_challenge_test.go b/core/security/password_challenge_test.go new file mode 100644 index 000000000..c2f68a145 --- /dev/null +++ b/core/security/password_challenge_test.go @@ -0,0 +1,108 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package security + +import "testing" + +// Setting a password should populate both the legacy bcrypt path and the new +// challenge-login material so either login mode can succeed afterward. +func TestSetPasswordPopulatesChallengeFields(t *testing.T) { + user := &UserAccount{} + if err := SetPassword(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("set password: %v", err) + } + + if user.Password == "" { + t.Fatal("expected password hash to be set") + } + if user.PasswordSalt == "" { + t.Fatal("expected password salt to be set") + } + if user.PasswordVerifier == "" { + t.Fatal("expected password verifier to be set") + } + if err := VerifyPassword(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("verify password: %v", err) + } +} + +// Existing challenge material should be stable when an account is already upgraded. +func TestEnsurePasswordChallengePreservesExistingVerifier(t *testing.T) { + user := &UserAccount{} + if err := SetPassword(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("set password: %v", err) + } + + originalSalt := user.PasswordSalt + originalVerifier := user.PasswordVerifier + if err := EnsurePasswordChallenge(user, "AnotherStrongPassw0rd!"); err != nil { + t.Fatalf("ensure password challenge: %v", err) + } + + if user.PasswordSalt != originalSalt { + t.Fatal("expected existing password salt to be preserved") + } + if user.PasswordVerifier != originalVerifier { + t.Fatal("expected existing password verifier to be preserved") + } +} + +// The framework wrapper should produce proofs compatible with the lower-level package. +func TestPasswordChallengeProofRoundTrip(t *testing.T) { + user := &UserAccount{} + login := "admin@example.org" + password := "StrongPassw0rd!" + + if err := SetPassword(user, password); err != nil { + t.Fatalf("set password: %v", err) + } + + challenge := NewLoginChallenge(login) + proof, err := BuildPasswordProof(user.PasswordVerifier, login, challenge.ID, challenge.Nonce) + if err != nil { + t.Fatalf("build password proof: %v", err) + } + + if !VerifyPasswordProof(user.PasswordVerifier, login, challenge.ID, challenge.Nonce, proof) { + t.Fatal("expected challenge proof to validate") + } +} + +// Legacy accounts that only had a bcrypt hash should become challenge-capable in place. +func TestEnsurePasswordChallengePopulatesLegacyAccount(t *testing.T) { + user := &UserAccount{Password: "existing-bcrypt-hash"} + if err := EnsurePasswordChallenge(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("ensure password challenge: %v", err) + } + + if user.PasswordSalt == "" { + t.Fatal("expected password salt to be populated") + } + if user.PasswordVerifier == "" { + t.Fatal("expected password verifier to be populated") + } + if !CanUsePasswordChallenge(user) { + t.Fatal("expected legacy account to become challenge-capable") + } +} diff --git a/core/security/passwordchallenge/password_challenge.go b/core/security/passwordchallenge/password_challenge.go index 461914b50..f16190bdb 100644 --- a/core/security/passwordchallenge/password_challenge.go +++ b/core/security/passwordchallenge/password_challenge.go @@ -37,24 +37,36 @@ import ( ) const ( - Method = "challenge" - Algorithm = "PBKDF2-SHA256" + // Method identifies the password challenge login flow returned by the challenge endpoint. + Method = "challenge" + // Algorithm describes the verifier/proof derivation algorithm that clients must use. + Algorithm = "PBKDF2-SHA256" + // Iterations is the PBKDF2 work factor shared with clients during challenge negotiation. Iterations = 120000 - keyLength = 32 + // keyLength is the derived key size used for both the stored verifier and request proof. + keyLength = 32 + // DefaultTTL is the default lifetime of a login challenge before it must be re-issued. DefaultTTL = 5 * time.Minute ) +// Challenge carries the one-time identifiers clients need to build a password proof locally. type Challenge struct { - ID string - Subject string - Nonce string + ID string + // Subject keeps the challenge bound to the login identity it was issued for. + Subject string + // Nonce is the random per-challenge input mixed into the client proof. + Nonce string + // ExpireAt marks when the one-time challenge stops being valid. ExpireAt time.Time } +// StoreOptions configures the lifetime of issued login challenges. type StoreOptions struct { + // TTL overrides the default challenge lifetime for this store instance. TTL time.Duration } +// Store tracks outstanding login challenges until they are consumed or expire. type Store struct { mu sync.Mutex ttl time.Duration @@ -63,6 +75,7 @@ type Store struct { var defaultStore = NewStore(StoreOptions{}) +// NewStore creates an in-memory challenge store with the requested TTL. func NewStore(options StoreOptions) *Store { ttl := options.TTL if ttl <= 0 { @@ -74,6 +87,7 @@ func NewStore(options StoreOptions) *Store { } } +// DeriveVerifier turns a password and salt into the verifier stored on the account record. func DeriveVerifier(password, salt string) (string, error) { if password == "" { return "", errors.New("password is empty") @@ -85,6 +99,7 @@ func DeriveVerifier(password, salt string) (string, error) { return hex.EncodeToString(key), nil } +// BuildProof derives the one-time challenge response that clients submit to /account/login. func BuildProof(verifier, subject, challengeID, nonce string) (string, error) { key, err := hex.DecodeString(verifier) if err != nil { @@ -99,6 +114,7 @@ func BuildProof(verifier, subject, challengeID, nonce string) (string, error) { return hex.EncodeToString(mac.Sum(nil)), nil } +// VerifyProof compares a submitted proof against the expected proof for this challenge tuple. func VerifyProof(verifier, subject, challengeID, nonce, proof string) bool { expected, err := BuildProof(verifier, subject, challengeID, nonce) if err != nil { @@ -115,14 +131,17 @@ func VerifyProof(verifier, subject, challengeID, nonce, proof string) bool { return hmac.Equal(expectedBytes, proofBytes) } +// New issues a login challenge from the default store. func New(subject string) Challenge { return defaultStore.New(subject) } +// Consume loads and invalidates a login challenge from the default store. func Consume(challengeID, subject string) (Challenge, error) { return defaultStore.Consume(challengeID, subject) } +// New allocates a fresh challenge for the provided subject. func (store *Store) New(subject string) Challenge { now := time.Now() store.mu.Lock() @@ -144,6 +163,7 @@ func (store *Store) New(subject string) Challenge { return challenge } +// Consume validates the subject and TTL, then invalidates the one-time challenge. func (store *Store) Consume(challengeID, subject string) (Challenge, error) { store.mu.Lock() defer store.mu.Unlock() diff --git a/core/security/passwordchallenge/password_challenge_test.go b/core/security/passwordchallenge/password_challenge_test.go index ed9eff938..8b0ce6aac 100644 --- a/core/security/passwordchallenge/password_challenge_test.go +++ b/core/security/passwordchallenge/password_challenge_test.go @@ -1,7 +1,35 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package passwordchallenge -import "testing" +import ( + "testing" + "time" +) +// Proof generation and verification need to round-trip because this package defines the +// wire contract shared between the framework login endpoint and upgraded clients. func TestPasswordChallengeProofRoundTrip(t *testing.T) { verifier, err := DeriveVerifier("admin", "salt-123") if err != nil { @@ -19,6 +47,7 @@ func TestPasswordChallengeProofRoundTrip(t *testing.T) { } } +// Challenges are bound to the requested login subject and must not be replayed for others. func TestConsumeRejectsWrongSubject(t *testing.T) { store := NewStore(StoreOptions{}) challenge := store.New("admin") @@ -27,3 +56,24 @@ func TestConsumeRejectsWrongSubject(t *testing.T) { t.Fatal("expected challenge subject mismatch to fail") } } + +// Empty input should be rejected up front to avoid persisting or comparing invalid verifiers. +func TestDeriveVerifierRejectsEmptyInput(t *testing.T) { + if _, err := DeriveVerifier("", "salt-123"); err == nil { + t.Fatal("expected empty password to fail") + } + if _, err := DeriveVerifier("admin", ""); err == nil { + t.Fatal("expected empty salt to fail") + } +} + +// Expiration keeps the one-time challenge store bounded and prevents stale proof reuse. +func TestConsumeRejectsExpiredChallenge(t *testing.T) { + store := NewStore(StoreOptions{TTL: time.Millisecond}) + challenge := store.New("admin") + + time.Sleep(5 * time.Millisecond) + if _, err := store.Consume(challenge.ID, "admin"); err == nil { + t.Fatal("expected expired challenge to fail") + } +} diff --git a/core/security/replay/replay.go b/core/security/replay/replay.go index 1619f0820..b5a630eaf 100644 --- a/core/security/replay/replay.go +++ b/core/security/replay/replay.go @@ -37,12 +37,16 @@ import ( ) const ( + // HeaderName is the HTTP header clients use to submit a one-time replay nonce. HeaderName = "X-Request-Nonce" + // DefaultTTL is the default lifetime of an issued replay nonce. DefaultTTL = 30 * time.Second ) +// SubjectExtractor derives the caller identity that a replay nonce should be bound to. type SubjectExtractor func(r *http.Request) string +// StoreOptions configures replay nonce retention and subject binding behavior. type StoreOptions struct { TTL time.Duration SubjectExtractor SubjectExtractor @@ -55,6 +59,7 @@ type nonceRecord struct { ExpiresAt time.Time } +// Store tracks issued replay nonces until they are consumed or expire. type Store struct { mu sync.Mutex ttl time.Duration @@ -64,6 +69,7 @@ type Store struct { var defaultStore = NewStore(StoreOptions{}) +// NewStore creates an in-memory replay store with optional TTL and subject extraction overrides. func NewStore(options StoreOptions) *Store { ttl := options.TTL if ttl <= 0 { @@ -80,14 +86,18 @@ func NewStore(options StoreOptions) *Store { } } +// IssueReplayNonce issues a nonce from the default store for the requested method/path scope. func IssueReplayNonce(r *http.Request, method, requestPath string) (string, time.Duration, error) { return defaultStore.IssueReplayNonce(r, method, requestPath) } +// ValidateAndConsumeReplayNonce validates a nonce from the default store and deletes it on success. func ValidateAndConsumeReplayNonce(r *http.Request) error { return defaultStore.ValidateAndConsumeReplayNonce(r) } +// DefaultSubjectExtractor binds anonymous callers together and authenticated callers to their +// Authorization header so replay nonces cannot be replayed across credential contexts. func DefaultSubjectExtractor(r *http.Request) string { if r == nil { return "anonymous" @@ -96,10 +106,13 @@ func DefaultSubjectExtractor(r *http.Request) string { if authorizationHeader == "" { return "anonymous" } + // Bind the nonce to the caller's authorization material so a replay token issued for one + // authenticated context cannot be reused with a different credential set. sum := sha256.Sum256([]byte(authorizationHeader)) return hex.EncodeToString(sum[:]) } +// IssueReplayNonce stores a nonce that is scoped to the caller, HTTP method, and request path. func (store *Store) IssueReplayNonce(r *http.Request, method, requestPath string) (string, time.Duration, error) { normalizedMethod, normalizedPath, err := normalizeScope(method, requestPath) if err != nil { @@ -126,6 +139,7 @@ func (store *Store) IssueReplayNonce(r *http.Request, method, requestPath string return nonce, store.ttl, nil } +// ValidateAndConsumeReplayNonce accepts a nonce only once and only for the original request scope. func (store *Store) ValidateAndConsumeReplayNonce(r *http.Request) error { if r == nil { return fmt.Errorf("request can not be nil") diff --git a/core/security/replay/replay_test.go b/core/security/replay/replay_test.go index b495b71c9..414cd79b5 100644 --- a/core/security/replay/replay_test.go +++ b/core/security/replay/replay_test.go @@ -1,11 +1,36 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + package replay import ( "net/http" "net/http/httptest" "testing" + "time" ) +// A nonce is one-time use by design, so any second validation attempt must fail. func TestReplayNonceCanOnlyBeUsedOnce(t *testing.T) { store := NewStore(StoreOptions{}) req := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) @@ -24,6 +49,7 @@ func TestReplayNonceCanOnlyBeUsedOnce(t *testing.T) { } } +// Replay tokens should stay bound to the caller identity, not just the raw path/method tuple. func TestReplayNonceBindsToAuthorizationHeader(t *testing.T) { store := NewStore(StoreOptions{}) issueReq := httptest.NewRequest(http.MethodPut, "https://console.local/credential/test", nil) @@ -42,6 +68,7 @@ func TestReplayNonceBindsToAuthorizationHeader(t *testing.T) { } } +// The scope includes HTTP method so a nonce issued for one mutation cannot authorize another. func TestReplayNonceBindsToPathAndMethod(t *testing.T) { store := NewStore(StoreOptions{}) issueReq := httptest.NewRequest(http.MethodPost, "https://console.local/setup/_initialize", nil) @@ -57,3 +84,45 @@ func TestReplayNonceBindsToPathAndMethod(t *testing.T) { t.Fatal("expected nonce with mismatched method to fail") } } + +// Anonymous callers still need a stable default subject so unauthenticated setup flows work. +func TestDefaultSubjectExtractorFallsBackToAnonymous(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) + if got := DefaultSubjectExtractor(req); got != "anonymous" { + t.Fatalf("expected anonymous subject, got %q", got) + } +} + +// Path normalization lets clients request a nonce with equivalent path forms safely. +func TestReplayNonceNormalizesPath(t *testing.T) { + store := NewStore(StoreOptions{}) + issueReq := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) + + nonce, _, err := store.IssueReplayNonce(issueReq, http.MethodPost, "account/../account/login") + if err != nil { + t.Fatalf("issue replay nonce failed: %v", err) + } + + useReq := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) + useReq.Header.Set(HeaderName, nonce) + if err := store.ValidateAndConsumeReplayNonce(useReq); err != nil { + t.Fatalf("expected normalized path to validate: %v", err) + } +} + +// Expired nonces should be rejected even if the caller, method, and path still match. +func TestReplayNonceExpires(t *testing.T) { + store := NewStore(StoreOptions{TTL: time.Millisecond}) + req := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) + + nonce, _, err := store.IssueReplayNonce(req, http.MethodPost, "/account/login") + if err != nil { + t.Fatalf("issue replay nonce failed: %v", err) + } + + time.Sleep(5 * time.Millisecond) + req.Header.Set(HeaderName, nonce) + if err := store.ValidateAndConsumeReplayNonce(req); err == nil { + t.Fatal("expected expired nonce to be rejected") + } +} diff --git a/core/security/session.go b/core/security/session.go index 612874241..dfbdf1593 100644 --- a/core/security/session.go +++ b/core/security/session.go @@ -6,12 +6,13 @@ package security import ( "fmt" - "github.com/golang-jwt/jwt" + "net/http" + "time" + + "github.com/golang-jwt/jwt/v4" "infini.sh/framework/core/api" "infini.sh/framework/core/errors" "infini.sh/framework/core/util" - "net/http" - "time" ) const UserAccessTokenSessionName = "user_session_access_token" diff --git a/core/security/user_profile.go b/core/security/user_profile.go index cd6936fe5..35155b0e2 100644 --- a/core/security/user_profile.go +++ b/core/security/user_profile.go @@ -33,10 +33,12 @@ type User struct { type UserAccount struct { orm.ORMObjectBase - Name string `json:"name,omitempty" elastic_mapping:"name: { type: keyword }" validate:"required" ` - Email string `json:"email,omitempty" elastic_mapping:"email: { type: keyword }" validate:"required|email" ` //unique - Roles []string `json:"roles,omitempty" elastic_mapping:"roles: { type: keyword }"` - Password string `json:"password,omitempty" elastic_mapping:"password: { type: keyword }"` + Name string `json:"name,omitempty" elastic_mapping:"name: { type: keyword }" validate:"required" ` + Email string `json:"email,omitempty" elastic_mapping:"email: { type: keyword }" validate:"required|email" ` //unique + Roles []string `json:"roles,omitempty" elastic_mapping:"roles: { type: keyword }"` + Password string `json:"password,omitempty" elastic_mapping:"password: { type: keyword }"` // Bcrypt hash used by the existing password-login flow. + PasswordSalt string `json:"password_salt,omitempty" elastic_mapping:"password_salt: { type: keyword }"` // Per-user salt exposed to clients during challenge login. + PasswordVerifier string `json:"password_verifier,omitempty" elastic_mapping:"password_verifier: { type: keyword }"` // Server-side verifier used to validate challenge proofs. } type UserProfile struct { diff --git a/core/security/user_session.go b/core/security/user_session.go index 9194299df..95e514745 100644 --- a/core/security/user_session.go +++ b/core/security/user_session.go @@ -25,13 +25,14 @@ package security import ( "fmt" + "time" + log "github.com/cihub/seelog" - "github.com/golang-jwt/jwt" + "github.com/golang-jwt/jwt/v4" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" "infini.sh/framework/core/param" "infini.sh/framework/core/util" - "time" ) type UserClaims struct { diff --git a/core/security/validate.go b/core/security/validate.go index a239c01eb..2dc3c1353 100644 --- a/core/security/validate.go +++ b/core/security/validate.go @@ -11,7 +11,7 @@ import ( "time" log "github.com/cihub/seelog" - "github.com/golang-jwt/jwt" + "github.com/golang-jwt/jwt/v4" "infini.sh/framework/core/errors" ) diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index f03337132..9996d3443 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -11,6 +11,7 @@ Information about release notes of INFINI Framework is provided here. ## Latest (In development) ### ❌ Breaking changes - refactor: native security are disabled by default #283 +- refactor: refactoring to simplify go modules #300 ### 🚀 Features - feat: support team-based scope for sharing services #258 @@ -19,12 +20,16 @@ Information about release notes of INFINI Framework is provided here. - feat(keystore): support large stdin secrets (>1024 bytes) and multiline #271 - feat(cors): add X-SERVICE-ID to allowed CORS headers #275 - feat: output HTTP access logs to file +- feat(metrics): monitor each disk and network interface independently for bottleneck detection +- feat(metrics): auto-detect network interface bandwidth per device (Linux, macOS, Windows) +- feat(metrics): identify specific bottleneck device (e.g., `disk_io:nvme0n1`, `network:eth0`) - feat(cookie): prevent aggressive session cookie expiration #284 - feat(client): support token-based authorization #288 - feat: add pluggable sink to host metrics collectors #288 +- feat: add access_token to security #359 +- feat(security): add native account login challenge, replay protection, and secure transport helpers ### 🐛 Bug fix -- fix: prevent duplicate bulk queue consumers during bulk indexing migrations #289 ### ✈️ Improvements - chore: API Handler Registration Improvements #283 - refactor: use PathUnescape to decode query param filter #249 @@ -43,7 +48,8 @@ Information about release notes of INFINI Framework is provided here. - chore: add util to get request header - chore: permissions refactoring - chore: security configuration structure enhanced - +- chore: remove unused grpc and cuckoo filter" +- chore: update seelog for vfs #363 ## 1.4.0 (2025-12-19) diff --git a/go.mod b/go.mod index 64788225d..65ac0c52e 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/go-ldap/ldap/v3 v3.4.11 github.com/go-redis/redis/v8 v8.11.5 github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f github.com/google/go-cmp v0.7.0 github.com/google/go-github v17.0.0+incompatible diff --git a/go.sum b/go.sum index ce0966a00..ce5b6c490 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg= github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f/go.mod h1:ijRvpgDJDI262hYq/IQVYgf8hd8IHUs93Ol0kvMBAx4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/modules/security/account/profile.go b/modules/security/account/profile.go index 3bb81aced..107da9a79 100644 --- a/modules/security/account/profile.go +++ b/modules/security/account/profile.go @@ -28,10 +28,21 @@ func Profile(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { } p := &security.UserProfile{ - Name: reqUser.Login, + Name: reqUser.Login, + Roles: reqUser.Roles, } p.ID = reqUser.UserID + if reqUser.Provider == security.DefaultNativeAuthBackend { + if _, account, err := security.GetUserByID(reqUser.UserID); err == nil && account != nil { + if account.Name != "" { + p.Name = account.Name + } + p.Email = account.Email + p.Roles = account.Roles + } + } + //get all permissions for user p.Permissions = security.GetAllPermissionsForUser(reqUser) diff --git a/modules/security/http_filters/json_mask.go b/modules/security/http_filters/json_mask.go index f98f88f6a..998b4d924 100644 --- a/modules/security/http_filters/json_mask.go +++ b/modules/security/http_filters/json_mask.go @@ -18,11 +18,13 @@ const FeatureRemoveSensitiveField = "feature_sensitive_fields_remove_sensitive_f const SensitiveFields = "feature_sensitive_fields_extra_keys" var sensitiveFields = map[string]bool{ - "password": true, - "token": true, - "secret": true, - "access_token": true, - "refresh_token": true, + "password": true, + "password_salt": true, + "password_verifier": true, + "token": true, + "secret": true, + "access_token": true, + "refresh_token": true, } type JSONMaskFilter struct{} diff --git a/modules/security/http_filters/security.go b/modules/security/http_filters/security.go new file mode 100644 index 000000000..6e372a000 --- /dev/null +++ b/modules/security/http_filters/security.go @@ -0,0 +1,75 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package http_filters + +import ( + "net/http" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + replaysecurity "infini.sh/framework/core/security/replay" +) + +func init() { + api.RegisterUIFilter(&SecurityFilter{}) +} + +// SecurityFilter enforces per-route HTTPS and replay-protection features declared in HandlerOptions. +type SecurityFilter struct { + api.Handler +} + +// GetPriority keeps the security checks ahead of permission checks but after early request shaping. +func (f *SecurityFilter) GetPriority() int { + return 450 +} + +// ApplyFilter translates route feature flags into runtime checks for HTTPS and replay nonce usage. +func (f *SecurityFilter) ApplyFilter( + method string, + pattern string, + options *api.HandlerOptions, + next httprouter.Handle, +) httprouter.Handle { + if options == nil || (!options.Feature(api.FeatureRequireSecureTransport) && !options.Feature(api.FeatureRequireReplayProtection)) { + log.Debug(method, ",", pattern, ", skip security feature filters") + return next + } + + return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + if options.Feature(api.FeatureRequireSecureTransport) { + secureOptions := api.SecureTransportOptions{ + TrustForwardHeaders: trustForwardHeadersFromOptions(options), + } + if !api.RequestUsesSecureTransport(r, secureOptions) { + f.WriteError(w, "this endpoint requires HTTPS. use https:// directly or route through a trusted HTTPS reverse proxy", http.StatusUpgradeRequired) + return + } + } + + if options.Feature(api.FeatureRequireReplayProtection) { + if err := replaysecurity.ValidateAndConsumeReplayNonce(r); err != nil { + f.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + } + + next(w, r, ps) + } +} + +// trustForwardHeadersFromOptions extracts whether SecureTransportOption opted into proxy headers. +func trustForwardHeadersFromOptions(options *api.HandlerOptions) bool { + if options == nil || options.Labels == nil { + return false + } + trustValue, ok := options.Labels[api.LabelTrustForwardHeaders] + if !ok { + return false + } + trustForwardHeaders, ok := trustValue.(bool) + return ok && trustForwardHeaders +} diff --git a/modules/security/http_filters/security_test.go b/modules/security/http_filters/security_test.go new file mode 100644 index 000000000..775463a43 --- /dev/null +++ b/modules/security/http_filters/security_test.go @@ -0,0 +1,128 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package http_filters + +import ( + "net/http" + "net/http/httptest" + "testing" + + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + replaysecurity "infini.sh/framework/core/security/replay" +) + +// Secure-transport enforcement should stop the request before the wrapped UI handler runs. +func TestSecurityFilterSecureTransportFeature(t *testing.T) { + filter := &SecurityFilter{} + options := &api.HandlerOptions{} + api.SecureTransportOption()(options) + + called := false + protected := filter.ApplyFilter(http.MethodPost, "/account/login", options, func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + called = true + w.WriteHeader(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodPost, "http://console.local/account/login", nil) + resp := httptest.NewRecorder() + protected(resp, req, nil) + + if called { + t.Fatal("expected insecure request to be blocked") + } + if resp.Code != http.StatusUpgradeRequired { + t.Fatalf("expected status %d, got %d", http.StatusUpgradeRequired, resp.Code) + } +} + +// When a nonce matches the request scope, the filter should behave like a no-op wrapper. +func TestSecurityFilterReplayProtectionFeature(t *testing.T) { + filter := &SecurityFilter{} + options := &api.HandlerOptions{} + api.ReplayProtectionOption()(options) + + req := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) + nonce, _, err := replaysecurity.IssueReplayNonce(req, http.MethodPost, "/account/login") + if err != nil { + t.Fatalf("issue replay nonce: %v", err) + } + req.Header.Set(replaysecurity.HeaderName, nonce) + + called := false + protected := filter.ApplyFilter(http.MethodPost, "/account/login", options, func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + called = true + w.WriteHeader(http.StatusOK) + }) + + resp := httptest.NewRecorder() + protected(resp, req, nil) + + if !called { + t.Fatal("expected replay-protected handler to run") + } + if resp.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, resp.Code) + } +} + +// Missing nonce headers must block replay-protected routes before business logic executes. +func TestSecurityFilterReplayProtectionRejectsMissingNonce(t *testing.T) { + filter := &SecurityFilter{} + options := &api.HandlerOptions{} + api.ReplayProtectionOption()(options) + + called := false + protected := filter.ApplyFilter(http.MethodPost, "/account/login", options, func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + called = true + w.WriteHeader(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodPost, "https://console.local/account/login", nil) + resp := httptest.NewRecorder() + protected(resp, req, nil) + + if called { + t.Fatal("expected missing nonce to block handler execution") + } + if resp.Code != http.StatusUnauthorized { + t.Fatalf("expected status %d, got %d", http.StatusUnauthorized, resp.Code) + } +} + +// Trusted forward headers let deployments behind HTTPS reverse proxies pass transport checks. +func TestSecurityFilterWithTrustedForwardHeaders(t *testing.T) { + filter := &SecurityFilter{} + options := &api.HandlerOptions{} + api.SecureTransportOption(api.SecureTransportOptions{TrustForwardHeaders: true})(options) + + called := false + protected := filter.ApplyFilter(http.MethodPost, "/account/login", options, func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + called = true + w.WriteHeader(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodPost, "http://console.local/account/login", nil) + req.Header.Set("X-Forwarded-Proto", "https") + resp := httptest.NewRecorder() + protected(resp, req, nil) + + if !called { + t.Fatal("expected trusted forwarded proto request to be allowed") + } + if resp.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, resp.Code) + } +} + +// Routes that do not opt into trusted proxy headers should stay conservative by default. +func TestTrustForwardHeadersFromOptionsDefaultsFalse(t *testing.T) { + if trustForwardHeadersFromOptions(nil) { + t.Fatal("expected nil options to disable trusted forward headers") + } + if trustForwardHeadersFromOptions(&api.HandlerOptions{}) { + t.Fatal("expected missing label to disable trusted forward headers") + } +} diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go new file mode 100644 index 000000000..1b0672800 --- /dev/null +++ b/modules/security/rbac/account_login.go @@ -0,0 +1,307 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package rbac + +import ( + "errors" + "net/http" + "strings" + "time" + + log "github.com/cihub/seelog" + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/security" + replaysecurity "infini.sh/framework/core/security/replay" + "infini.sh/framework/core/util" +) + +var ( + // Keep the password and challenge paths aligned on one user-facing failure message. + errInvalidLoginCredentials = errors.New("invalid login or password") + // A challenge login must send both the one-time challenge id and the derived proof. + errIncompleteChallenge = errors.New("challenge response is incomplete") + // Password login keeps requiring the legacy password field when no challenge proof is supplied. + errMissingPassword = errors.New("password is required") +) + +// accountLoginRequest accepts both the framework-native "login" field and the aliases +// already used by existing clients while challenge login is rolled out incrementally. +type accountLoginRequest struct { + Login string `json:"login"` + Email string `json:"email"` + Username string `json:"username"` + UserName string `json:"userName"` + Password string `json:"password"` + ChallengeID string `json:"challenge_id"` + Proof string `json:"proof"` +} + +func registerAccountRoutes() { + // These endpoints are only registered from rbac.Init(), so they exist only when + // native authentication is enabled and the native user backend is ready. + api.HandleUIMethod(api.POST, "/account/replay_nonce", + api.RequireSecureTransport(IssueReplayNonce), + api.AllowPublicAccess(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS)) + + api.HandleUIMethod(api.POST, "/account/login/challenge", + api.RequireSecureTransport(LoginChallenge), + api.AllowPublicAccess(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS)) + + api.HandleUIMethod(api.POST, "/account/login", + api.RequireSecureTransport(Login), + api.AllowPublicAccess(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS)) +} + +// IssueReplayNonce mints a short-lived nonce bound to the caller and target request scope. +func IssueReplayNonce(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + var req struct { + Method string `json:"method"` + Path string `json:"path"` + } + + if err := api.DecodeJSON(r, &req); err != nil { + api.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + + nonce, ttl, err := replaysecurity.IssueReplayNonce(r, req.Method, req.Path) + if err != nil { + api.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + + api.WriteOKJSON(w, util.MapStr{ + "status": "ok", + "nonce": nonce, + "expire_in_seconds": int(ttl / time.Second), + }) +} + +// LoginChallenge tells the client whether this account can use challenge login and, if so, +// returns the one-time challenge payload required to derive the proof locally. +func LoginChallenge(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + var req accountLoginRequest + if err := api.DecodeJSON(r, &req); err != nil { + api.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + + login := req.NormalizedLogin() + if login == "" { + api.WriteError(w, "login is required", http.StatusBadRequest) + return + } + + exists, user, err := lookupAccountByLogin(login) + if err != nil { + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + + api.WriteOKJSON(w, buildLoginChallengeResponse(login, exists, user)) +} + +// Login accepts either the legacy password payload or the new challenge proof and then +// reuses the existing session/token issuance path once the credentials are verified. +func Login(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + var req accountLoginRequest + if err := api.DecodeJSON(r, &req); err != nil { + api.WriteError(w, err.Error(), http.StatusBadRequest) + return + } + + login := req.NormalizedLogin() + if login == "" { + api.WriteError(w, "login is required", http.StatusBadRequest) + return + } + + exists, user, err := lookupAccountByLogin(login) + if err != nil { + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !exists || user == nil { + api.WriteError(w, errInvalidLoginCredentials.Error(), http.StatusForbidden) + return + } + + usedChallenge := req.ChallengeID != "" || req.Proof != "" + if err := validateReplayNonce(r, usedChallenge); err != nil { + api.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + + usedChallenge, err = authenticateLogin(user, login, req.Password, req.ChallengeID, req.Proof) + if err != nil { + statusCode := http.StatusForbidden + if errors.Is(err, errIncompleteChallenge) || errors.Is(err, errMissingPassword) { + statusCode = http.StatusBadRequest + } + api.WriteError(w, err.Error(), statusCode) + return + } + + if !usedChallenge { + upgradePasswordChallenge(user, req.Password) + } + + sessionUser := newNativeSession(user, login) + if err, token := security.AddUserToSession(w, r, sessionUser); err != nil { + api.WriteError(w, err.Error(), http.StatusInternalServerError) + } else { + api.WriteOKJSON(w, token) + } +} + +// NormalizedLogin resolves the various historical request field names into one lookup key. +func (req accountLoginRequest) NormalizedLogin() string { + for _, candidate := range []string{req.Login, req.Email, req.Username, req.UserName} { + if value := strings.TrimSpace(candidate); value != "" { + return value + } + } + return "" +} + +// buildLoginChallengeResponse keeps the challenge negotiation explicit: challenge-capable +// accounts get the proof derivation inputs, while older accounts stay on plain login. +func buildLoginChallengeResponse(login string, exists bool, user *security.UserAccount) util.MapStr { + if exists && security.CanUsePasswordChallenge(user) { + // The challenge payload gives clients everything needed to derive a proof + // locally without sending the raw password back to the server. + challenge := security.NewLoginChallenge(login) + return util.MapStr{ + "status": "ok", + "method": security.PasswordChallengeMethod, + "algorithm": security.PasswordChallengeAlgorithm, + "iterations": security.PasswordChallengeIterations, + "challenge_id": challenge.ID, + "nonce": challenge.Nonce, + "salt": user.PasswordSalt, + } + } + + return util.MapStr{ + "status": "ok", + "method": "plain", + } +} + +// authenticateLogin selects the correct credential validation path based on the request body. +func authenticateLogin(user *security.UserAccount, login, password, challengeID, proof string) (bool, error) { + if user == nil { + return false, errInvalidLoginCredentials + } + + if challengeID != "" || proof != "" { + if challengeID == "" || proof == "" { + return true, errIncompleteChallenge + } + + challenge, err := security.ConsumeLoginChallenge(challengeID, login) + if err != nil || !security.CanUsePasswordChallenge(user) { + return true, errInvalidLoginCredentials + } + if !security.VerifyPasswordProof(user.PasswordVerifier, login, challenge.ID, challenge.Nonce, proof) { + return true, errInvalidLoginCredentials + } + return true, nil + } + + if password == "" { + return false, errMissingPassword + } + if err := security.VerifyPassword(user, password); err != nil { + return false, errInvalidLoginCredentials + } + return false, nil +} + +// lookupAccountByLogin normalizes the service-registry "not found" result into a regular miss. +func lookupAccountByLogin(login string) (bool, *security.UserAccount, error) { + exists, user, err := GetUserByLogin(login) + if err != nil && err.Error() == "not found" { + return false, nil, nil + } + return exists, user, err +} + +// validateReplayNonce keeps challenge login replay-safe while leaving older password-only +// clients working until they adopt the explicit nonce negotiation endpoint. +func validateReplayNonce(r *http.Request, required bool) error { + nonce := strings.TrimSpace(r.Header.Get(replaysecurity.HeaderName)) + if nonce == "" && !required { + // Keep the original password login path backward compatible: upgraded clients + // send replay nonces, while older clients can still post passwords directly. + return nil + } + return replaysecurity.ValidateAndConsumeReplayNonce(r) +} + +// upgradePasswordChallenge backfills verifier material after a successful legacy login so +// existing native accounts can move onto the challenge flow without an offline migration. +func upgradePasswordChallenge(user *security.UserAccount, password string) { + if user == nil || password == "" || security.CanUsePasswordChallenge(user) { + return + } + + if err := security.EnsurePasswordChallenge(user, password); err != nil { + log.Warnf("failed to derive password challenge for user [%s]: %v", user.Email, err) + return + } + + // Persist the verifier after a successful legacy password login so subsequent + // logins can move onto the challenge flow without an explicit migration step. + ctx := orm.NewContext() + ctx.DirectAccess() + ctx.Refresh = orm.WaitForRefresh + if err := orm.Update(ctx, user); err != nil { + log.Warnf("failed to persist password challenge for user [%s]: %v", user.Email, err) + } +} + +// newNativeSession converts a native account record into the existing framework session claims. +func newNativeSession(user *security.UserAccount, login string) *security.UserSessionInfo { + userLogin := strings.TrimSpace(user.Email) + if userLogin == "" { + userLogin = login + } + + session := &security.UserSessionInfo{ + Provider: security.DefaultNativeAuthBackend, + Login: userLogin, + Roles: append([]string(nil), user.Roles...), + } + session.SetUserID(user.ID) + return session +} diff --git a/modules/security/rbac/account_login_test.go b/modules/security/rbac/account_login_test.go new file mode 100644 index 000000000..fe66941db --- /dev/null +++ b/modules/security/rbac/account_login_test.go @@ -0,0 +1,190 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package rbac + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + + "infini.sh/framework/core/security" + replaysecurity "infini.sh/framework/core/security/replay" +) + +// The request payload accepts multiple historical login field names during rollout. +func TestAccountLoginRequestNormalizedLogin(t *testing.T) { + req := accountLoginRequest{ + Email: "admin@example.org", + Username: "ignored@example.org", + } + + if got := req.NormalizedLogin(); got != "admin@example.org" { + t.Fatalf("expected email to be preferred, got %q", got) + } +} + +// Password login remains the backward-compatible path for accounts and clients not yet upgraded. +func TestAuthenticateLoginWithPassword(t *testing.T) { + user := &security.UserAccount{Email: "admin@example.org"} + if err := security.SetPassword(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("set password: %v", err) + } + + usedChallenge, err := authenticateLogin(user, user.Email, "StrongPassw0rd!", "", "") + if err != nil { + t.Fatalf("authenticate login: %v", err) + } + if usedChallenge { + t.Fatal("expected password login path") + } +} + +// Challenge login should succeed once the account already has verifier material. +func TestAuthenticateLoginWithChallenge(t *testing.T) { + user := &security.UserAccount{Email: "admin@example.org"} + if err := security.SetPassword(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("set password: %v", err) + } + + challenge := security.NewLoginChallenge(user.Email) + proof, err := security.BuildPasswordProof(user.PasswordVerifier, user.Email, challenge.ID, challenge.Nonce) + if err != nil { + t.Fatalf("build password proof: %v", err) + } + + usedChallenge, err := authenticateLogin(user, user.Email, "", challenge.ID, proof) + if err != nil { + t.Fatalf("authenticate login: %v", err) + } + if !usedChallenge { + t.Fatal("expected challenge login path") + } +} + +// Partially supplied challenge payloads should fail distinctly from bad credentials. +func TestAuthenticateLoginRejectsIncompleteChallenge(t *testing.T) { + user := &security.UserAccount{Email: "admin@example.org"} + if err := security.SetPassword(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("set password: %v", err) + } + + _, err := authenticateLogin(user, user.Email, "", "challenge-id", "") + if !errors.Is(err, errIncompleteChallenge) { + t.Fatalf("expected incomplete challenge error, got %v", err) + } +} + +// Incorrect proofs should collapse to the same user-facing error as bad passwords. +func TestAuthenticateLoginRejectsWrongProof(t *testing.T) { + user := &security.UserAccount{Email: "admin@example.org"} + if err := security.SetPassword(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("set password: %v", err) + } + + challenge := security.NewLoginChallenge(user.Email) + _, err := authenticateLogin(user, user.Email, "", challenge.ID, "bad-proof") + if !errors.Is(err, errInvalidLoginCredentials) { + t.Fatalf("expected invalid credential error, got %v", err) + } +} + +// Older accounts intentionally advertise plain login until their verifier is available. +func TestBuildLoginChallengeResponseFallsBackToPlain(t *testing.T) { + user := &security.UserAccount{Email: "admin@example.org"} + resp := buildLoginChallengeResponse(user.Email, true, user) + + if got := resp["method"]; got != "plain" { + t.Fatalf("expected plain fallback, got %v", got) + } + if _, ok := resp["challenge_id"]; ok { + t.Fatal("did not expect challenge payload for plain fallback") + } +} + +// Upgraded accounts should return the exact challenge inputs the client needs next. +func TestBuildLoginChallengeResponseReturnsChallenge(t *testing.T) { + user := &security.UserAccount{Email: "admin@example.org"} + if err := security.SetPassword(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("set password: %v", err) + } + + resp := buildLoginChallengeResponse(user.Email, true, user) + if got := resp["method"]; got != security.PasswordChallengeMethod { + t.Fatalf("expected challenge method, got %v", got) + } + if resp["challenge_id"] == "" { + t.Fatal("expected challenge id to be returned") + } + if resp["nonce"] == "" { + t.Fatal("expected nonce to be returned") + } + if resp["salt"] != user.PasswordSalt { + t.Fatal("expected challenge response to expose password salt") + } +} + +// Legacy password clients keep working even before they learn the replay-nonce preflight. +func TestValidateReplayNonceAllowsLegacyPasswordLoginWithoutNonce(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/account/login", nil) + if err := validateReplayNonce(req, false); err != nil { + t.Fatalf("expected missing nonce to be allowed for legacy password login, got %v", err) + } +} + +// Challenge logins must enforce nonce usage immediately because the frontend already negotiated it. +func TestValidateReplayNonceRequiresNonceForChallengeLogin(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/account/login", nil) + if err := validateReplayNonce(req, true); err == nil { + t.Fatal("expected missing nonce to be rejected for challenge login") + } +} + +// Once a nonce is explicitly issued for /account/login it should validate on that exact route. +func TestValidateReplayNonceConsumesIssuedNonce(t *testing.T) { + req := httptest.NewRequest(http.MethodPost, "/account/login", nil) + nonce, _, err := replaysecurity.IssueReplayNonce(req, http.MethodPost, "/account/login") + if err != nil { + t.Fatalf("issue replay nonce: %v", err) + } + req.Header.Set(replaysecurity.HeaderName, nonce) + + if err := validateReplayNonce(req, true); err != nil { + t.Fatalf("expected issued nonce to validate, got %v", err) + } +} + +// Native sessions should still be constructible even when the stored account email is blank. +func TestNewNativeSessionFallsBackToRequestedLogin(t *testing.T) { + user := &security.UserAccount{Email: "", Roles: []string{security.RoleAdmin}} + user.ID = "user-1" + + session := newNativeSession(user, "admin@example.org") + if session.Login != "admin@example.org" { + t.Fatalf("expected requested login fallback, got %q", session.Login) + } + if session.Provider != security.DefaultNativeAuthBackend { + t.Fatalf("expected native provider, got %q", session.Provider) + } +} diff --git a/modules/security/rbac/init.go b/modules/security/rbac/init.go index 4caa07f65..1a2a022ab 100644 --- a/modules/security/rbac/init.go +++ b/modules/security/rbac/init.go @@ -18,6 +18,7 @@ func Init() { provider := SecurityBackendProvider{} security.RegisterAuthenticationProvider(security.DefaultNativeAuthBackend, &provider) security.RegisterAuthorizationProvider(security.DefaultNativeAuthBackend, &provider) + registerAccountRoutes() orm.MustRegisterSchemaWithIndexName(&security.UserAccount{}, "app-users") orm.MustRegisterSchemaWithIndexName(&security.UserRole{}, "app-roles") diff --git a/modules/security/rbac/user.go b/modules/security/rbac/user.go index 569bc1917..64d36b44e 100644 --- a/modules/security/rbac/user.go +++ b/modules/security/rbac/user.go @@ -8,7 +8,6 @@ import ( "net/http" log "github.com/cihub/seelog" - "golang.org/x/crypto/bcrypt" "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" @@ -73,16 +72,18 @@ func UpdateUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) } if obj.Password == "" { + // Preserve the verifier material on metadata-only updates so editing roles, + // names, or other fields does not silently disable challenge login. obj.Password = oldObj.Password + obj.PasswordSalt = oldObj.PasswordSalt + obj.PasswordVerifier = oldObj.PasswordVerifier } else { if !util.ValidateSecure(obj.Password) { panic("should be secured password") } - hash, err := bcrypt.GenerateFromPassword([]byte(obj.Password), bcrypt.DefaultCost) - if err != nil { + if err := security.SetPassword(&obj, obj.Password); err != nil { panic(err) } - obj.Password = string(hash) } ctx.Refresh = orm.WaitForRefresh err = orm.Update(ctx, &obj) @@ -204,17 +205,15 @@ func (provider *SecurityBackendProvider) CreateUser(name, email, password string log.Warn("email already exists, will be replaced") obj.ID = account.ID } else { - obj.ID = getUIDByEmail(obj.Email) + obj.ID = getUIDByEmail(email) } - hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) - if err != nil { - panic(err) - } obj.Name = name obj.Email = email obj.Roles = []string{security.RoleAdmin} - obj.Password = string(hash) + if err := security.SetPassword(obj, password); err != nil { + panic(err) + } ctx := orm.NewContext() ctx.DirectAccess() @@ -252,13 +251,10 @@ func CreateUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) } randStr := util.GenerateSecureString(8) - hash, err := bcrypt.GenerateFromPassword([]byte(randStr), bcrypt.DefaultCost) - if err != nil { + if err := security.SetPassword(obj, randStr); err != nil { panic(err) } - obj.Password = string(hash) - ctx := orm.NewContextWithParent(req.Context()) ctx.Refresh = orm.WaitForRefresh err = orm.Save(ctx, obj) @@ -267,5 +263,9 @@ func CreateUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) } obj.Password = randStr + // The one-time bootstrap password should be returned to the caller, but the + // persisted verifier material must stay server-side only. + obj.PasswordSalt = "" + obj.PasswordVerifier = "" api.WriteJSON(w, obj, 200) } From 9f276ceeb5274c61d4e044ed3ebc267f915cd58e Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 26 May 2026 17:06:14 +0800 Subject: [PATCH 052/137] feat: support shared account flow migration\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- core/security/account_flow.go | 88 ++++++++++++++++++ core/security/password_challenge.go | 41 ++++++--- core/security/session.go | 46 +++++++++- core/security/user_session.go | 63 +++++++++++++ core/security/user_session_test.go | 88 ++++++++++++++++++ core/security/validate.go | 21 +++-- core/security/validate_test.go | 68 ++++++++++++++ modules/security/account/refresh.go | 95 ++++++++++++++++++++ modules/security/account/refresh_test.go | 98 +++++++++++++++++++++ modules/security/rbac/account_login.go | 49 +++++++---- modules/security/rbac/account_login_test.go | 85 +++++++++++++++++- 11 files changed, 702 insertions(+), 40 deletions(-) create mode 100644 core/security/account_flow.go create mode 100644 core/security/user_session_test.go create mode 100644 core/security/validate_test.go create mode 100644 modules/security/account/refresh.go create mode 100644 modules/security/account/refresh_test.go diff --git a/core/security/account_flow.go b/core/security/account_flow.go new file mode 100644 index 000000000..9cf0e886c --- /dev/null +++ b/core/security/account_flow.go @@ -0,0 +1,88 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package security + +import "sync" + +// AccountPasswordLoginProvider lets applications keep their own password-auth realms +// while reusing the framework-owned /account/login HTTP flow and session issuance. +type AccountPasswordLoginProvider interface { + AuthenticateByPassword(login, password string) (*UserSessionInfo, error) +} + +var accountPasswordLoginProviders = sync.Map{} + +func RegisterAccountPasswordLoginProvider(name string, provider AccountPasswordLoginProvider) { + accountPasswordLoginProviders.Store(name, provider) +} + +// AuthenticateAccountPasswordLogin tries application-provided password login providers +// after the native framework account path has either not matched or not succeeded. +func AuthenticateAccountPasswordLogin(login, password string) (*UserSessionInfo, error) { + var out *UserSessionInfo + var lastErr error + + accountPasswordLoginProviders.Range(func(key, value any) bool { + provider, ok := value.(AccountPasswordLoginProvider) + if !ok { + return true + } + + sessionUser, err := provider.AuthenticateByPassword(login, password) + if err != nil { + lastErr = err + return true + } + if sessionUser != nil { + out = sessionUser + return false + } + return true + }) + + if out != nil { + return out, nil + } + return nil, lastErr +} + +// SessionTokenResponseDecorator lets applications enrich the shared login/refresh +// response with app-specific fields while reusing the framework session pipeline. +type SessionTokenResponseDecorator func(token map[string]interface{}, user *UserSessionInfo) + +var sessionTokenResponseDecorators = sync.Map{} + +func RegisterSessionTokenResponseDecorator(name string, decorator SessionTokenResponseDecorator) { + sessionTokenResponseDecorators.Store(name, decorator) +} + +func applySessionTokenResponseDecorators(token map[string]interface{}, user *UserSessionInfo) { + sessionTokenResponseDecorators.Range(func(key, value any) bool { + decorator, ok := value.(SessionTokenResponseDecorator) + if ok { + decorator(token, user) + } + return true + }) +} diff --git a/core/security/password_challenge.go b/core/security/password_challenge.go index 2c860a6ed..c007031cd 100644 --- a/core/security/password_challenge.go +++ b/core/security/password_challenge.go @@ -43,33 +43,54 @@ const ( // LoginChallenge re-exports the framework challenge payload used by native account login. type LoginChallenge = passwordchallenge.Challenge +// PasswordMaterial bundles the fields that apps need to persist after accepting a password. +type PasswordMaterial struct { + Hash string + Salt string + Verifier string +} + // CanUsePasswordChallenge reports whether a native account already has challenge credentials. func CanUsePasswordChallenge(user *UserAccount) bool { return user != nil && user.PasswordSalt != "" && user.PasswordVerifier != "" } -// SetPassword updates both the legacy bcrypt hash and the challenge verifier material. -func SetPassword(user *UserAccount, password string) error { - if user == nil { - return errors.New("user is nil") - } - +// GeneratePasswordMaterial derives the bcrypt hash and challenge verifier fields for a password. +func GeneratePasswordMaterial(password string) (*PasswordMaterial, error) { hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { - return err + return nil, err } // Store both the bcrypt hash for existing password checks and the derived // verifier for challenge login so the two login modes stay in sync. salt := util.GenerateSecureString(32) verifier, err := DerivePasswordVerifier(password, salt) + if err != nil { + return nil, err + } + + return &PasswordMaterial{ + Hash: string(hash), + Salt: salt, + Verifier: verifier, + }, nil +} + +// SetPassword updates both the legacy bcrypt hash and the challenge verifier material. +func SetPassword(user *UserAccount, password string) error { + if user == nil { + return errors.New("user is nil") + } + + material, err := GeneratePasswordMaterial(password) if err != nil { return err } - user.Password = string(hash) - user.PasswordSalt = salt - user.PasswordVerifier = verifier + user.Password = material.Hash + user.PasswordSalt = material.Salt + user.PasswordVerifier = material.Verifier return nil } diff --git a/core/security/session.go b/core/security/session.go index dfbdf1593..6f52e016e 100644 --- a/core/security/session.go +++ b/core/security/session.go @@ -16,6 +16,7 @@ import ( ) const UserAccessTokenSessionName = "user_session_access_token" +const UserAccessTokenTTL = 24 * time.Hour func init() { RegisterHTTPAuthFilterProvider("session_token", byAccessTokenSession) @@ -89,7 +90,7 @@ func GenerateJWTAccessToken(user *UserSessionInfo) (map[string]interface{}, erro token1 := jwt.NewWithClaims(jwt.SigningMethodHS256, UserClaims{ UserSessionInfo: user, RegisteredClaims: &jwt.RegisteredClaims{ - ExpiresAt: jwt.NewNumericDate(time.Now().Add(24 * time.Hour)), + ExpiresAt: jwt.NewNumericDate(time.Now().Add(UserAccessTokenTTL)), }, }) @@ -105,7 +106,7 @@ func GenerateJWTAccessToken(user *UserSessionInfo) (map[string]interface{}, erro data = util.MapStr{ "access_token": tokenString, - "expire_in": time.Now().Unix() + 86400, //24h + "expire_in": time.Now().Unix() + int64(UserAccessTokenTTL/time.Second), } data["status"] = "ok" @@ -113,3 +114,44 @@ func GenerateJWTAccessToken(user *UserSessionInfo) (map[string]interface{}, erro return data, err } + +// DecorateSessionTokenResponse keeps framework-issued account responses directly +// consumable by existing console clients while auth flows converge on framework. +func DecorateSessionTokenResponse(token map[string]interface{}, user *UserSessionInfo) { + if token == nil || user == nil { + return + } + + if expiresAt := tokenExpiresAtUnix(token["expire_in"]); expiresAt > 0 { + token["expires_at"] = expiresAt + + remaining := expiresAt - time.Now().Unix() + if remaining < 0 { + remaining = 0 + } + token["expire_in"] = remaining + } + + token["username"] = user.Login + token["id"] = user.UserID + token["roles"] = append([]string(nil), user.Roles...) + token["privilege"] = GetAllPermissionsForUser(user) + applySessionTokenResponseDecorators(token, user) +} + +func tokenExpiresAtUnix(value interface{}) int64 { + switch v := value.(type) { + case int64: + return v + case int: + return int64(v) + case int32: + return int64(v) + case float64: + return int64(v) + case float32: + return int64(v) + default: + return 0 + } +} diff --git a/core/security/user_session.go b/core/security/user_session.go index 95e514745..0bbcddfb8 100644 --- a/core/security/user_session.go +++ b/core/security/user_session.go @@ -24,7 +24,9 @@ package security import ( + "encoding/json" "fmt" + "strings" "time" log "github.com/cihub/seelog" @@ -47,6 +49,67 @@ func NewUserClaims() *UserClaims { } } +type userSessionInfoAlias UserSessionInfo + +// MarshalJSON keeps the framework claims readable by older console clients while the +// token/session stack is converging onto the shared framework implementation. +func (c UserClaims) MarshalJSON() ([]byte, error) { + sessionUser := c.UserSessionInfo + if sessionUser == nil { + sessionUser = &UserSessionInfo{} + } + + claims := c.RegisteredClaims + if claims == nil { + claims = &jwt.RegisteredClaims{} + } + + return json.Marshal(struct { + *jwt.RegisteredClaims + *userSessionInfoAlias + Username string `json:"username,omitempty"` + UserID string `json:"user_id,omitempty"` + }{ + RegisteredClaims: claims, + userSessionInfoAlias: (*userSessionInfoAlias)(sessionUser), + Username: sessionUser.Login, + UserID: sessionUser.UserID, + }) +} + +// UnmarshalJSON accepts both the framework-native field names and the older console +// aliases so apps can switch validators without forcing a token-format fork first. +func (c *UserClaims) UnmarshalJSON(data []byte) error { + aux := struct { + *jwt.RegisteredClaims + *userSessionInfoAlias + Username string `json:"username,omitempty"` + UserID string `json:"user_id,omitempty"` + }{ + RegisteredClaims: &jwt.RegisteredClaims{}, + userSessionInfoAlias: &userSessionInfoAlias{}, + } + + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + sessionUser := (*UserSessionInfo)(aux.userSessionInfoAlias) + if sessionUser == nil { + sessionUser = &UserSessionInfo{} + } + if strings.TrimSpace(sessionUser.Login) == "" { + sessionUser.Login = strings.TrimSpace(aux.Username) + } + if strings.TrimSpace(sessionUser.UserID) == "" { + sessionUser.UserID = strings.TrimSpace(aux.UserID) + } + + c.RegisteredClaims = aux.RegisteredClaims + c.UserSessionInfo = sessionUser + return nil +} + // auth user info type UserSessionInfo struct { param.Parameters diff --git a/core/security/user_session_test.go b/core/security/user_session_test.go new file mode 100644 index 000000000..41b37eb19 --- /dev/null +++ b/core/security/user_session_test.go @@ -0,0 +1,88 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package security + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/golang-jwt/jwt/v4" +) + +// Framework-issued tokens need to remain readable by console clients until the two +// stacks finish converging on a single session claim format. +func TestUserClaimsMarshalIncludesLegacyConsoleAliases(t *testing.T) { + claims := UserClaims{ + RegisteredClaims: &jwt.RegisteredClaims{}, + UserSessionInfo: &UserSessionInfo{ + Provider: "native", + Login: "admin@example.org", + Roles: []string{RoleAdmin}, + UserID: "user-1", + }, + } + + payload, err := json.Marshal(claims) + if err != nil { + t.Fatalf("marshal claims: %v", err) + } + + text := string(payload) + for _, expected := range []string{ + `"login":"admin@example.org"`, + `"username":"admin@example.org"`, + `"userid":"user-1"`, + `"user_id":"user-1"`, + } { + if !strings.Contains(text, expected) { + t.Fatalf("expected %s in %s", expected, text) + } + } +} + +// Older console tokens only carried username/user_id, so the framework parser must +// backfill its native login/userid fields from those aliases during migration. +func TestUserClaimsUnmarshalAcceptsLegacyConsoleAliases(t *testing.T) { + var claims UserClaims + err := json.Unmarshal([]byte(`{ + "provider":"native", + "username":"admin@example.org", + "user_id":"user-1", + "roles":["admin"] + }`), &claims) + if err != nil { + t.Fatalf("unmarshal claims: %v", err) + } + + if claims.Login != "admin@example.org" { + t.Fatalf("expected login to be backfilled from username, got %q", claims.Login) + } + if claims.UserID != "user-1" { + t.Fatalf("expected user id to be backfilled from user_id, got %q", claims.UserID) + } + if claims.Provider != "native" { + t.Fatalf("expected provider to be preserved, got %q", claims.Provider) + } +} diff --git a/core/security/validate.go b/core/security/validate.go index 2dc3c1353..0ae2ce4e0 100644 --- a/core/security/validate.go +++ b/core/security/validate.go @@ -15,11 +15,8 @@ import ( "infini.sh/framework/core/errors" ) -func byAuthorizationHeader(w http.ResponseWriter, r *http.Request) (claims *UserClaims, err error) { - var ( - authorization = r.Header.Get("Authorization") - ok bool - ) +func parseUserClaimsFromAuthorizationHeader(authorization string) (claims *UserClaims, err error) { + var ok bool if authorization == "" { return nil, errors.Error("Authorization not found") @@ -63,6 +60,20 @@ func byAuthorizationHeader(w http.ResponseWriter, r *http.Request) (claims *User return claims, nil } +// ValidateAuthorizationHeader validates a bearer token header and returns the +// decoded framework session information for callers that only have header access. +func ValidateAuthorizationHeader(authorization string) (*UserSessionInfo, error) { + claims, err := parseUserClaimsFromAuthorizationHeader(authorization) + if err != nil { + return nil, err + } + return claims.UserSessionInfo, nil +} + +func byAuthorizationHeader(w http.ResponseWriter, r *http.Request) (claims *UserClaims, err error) { + return parseUserClaimsFromAuthorizationHeader(r.Header.Get("Authorization")) +} + func ValidateLogin(w http.ResponseWriter, r *http.Request) (session *UserSessionInfo, err error) { var claims *UserClaims diff --git a/core/security/validate_test.go b/core/security/validate_test.go new file mode 100644 index 000000000..144a6f8c5 --- /dev/null +++ b/core/security/validate_test.go @@ -0,0 +1,68 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package security + +import ( + "testing" + "time" + + "github.com/golang-jwt/jwt/v4" +) + +// Header-only callers in downstream apps need the same validation path as the +// framework HTTP auth middleware while shared auth code is being adopted. +func TestValidateAuthorizationHeader(t *testing.T) { + oldSecret := secretKey + secretKey = "test-framework-secret" + defer func() { + secretKey = oldSecret + }() + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, UserClaims{ + RegisteredClaims: &jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour)), + }, + UserSessionInfo: &UserSessionInfo{ + Provider: "native", + Login: "admin@example.org", + Roles: []string{RoleAdmin}, + UserID: "user-1", + }, + }) + tokenString, err := token.SignedString([]byte(secretKey)) + if err != nil { + t.Fatalf("sign token: %v", err) + } + + sessionUser, err := ValidateAuthorizationHeader("Bearer " + tokenString) + if err != nil { + t.Fatalf("validate authorization header: %v", err) + } + if sessionUser.Login != "admin@example.org" { + t.Fatalf("expected login to round-trip, got %q", sessionUser.Login) + } + if sessionUser.UserID != "user-1" { + t.Fatalf("expected user id to round-trip, got %q", sessionUser.UserID) + } +} diff --git a/modules/security/account/refresh.go b/modules/security/account/refresh.go new file mode 100644 index 000000000..ea000d9e9 --- /dev/null +++ b/modules/security/account/refresh.go @@ -0,0 +1,95 @@ +/* Copyright © INFINI LTD. All rights reserved. + * Web: https://infinilabs.com + * Email: hello#infini.ltd */ + +package account + +import ( + "fmt" + "net/http" + "strings" + + "infini.sh/framework/core/api" + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/security" +) + +func init() { + api.HandleUIMethod(api.POST, "/account/refresh", api.RequireSecureTransport(Refresh), api.RequireLogin(), api.AllowOPTIONSS(), api.Feature(api.FeatureCORS)) +} + +// Refresh reissues an access token for the current session user while reloading the +// native account record so updated roles/profile data are reflected in new tokens. +func Refresh(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + reqUser, err := security.GetUserFromContext(r.Context()) + if err != nil || reqUser == nil { + api.WriteError(w, "invalid user", http.StatusUnauthorized) + return + } + + sessionUser, err := buildRefreshedSession(reqUser) + if err != nil { + api.WriteError(w, err.Error(), http.StatusUnauthorized) + return + } + + if err, token := security.AddUserToSession(w, r, sessionUser); err != nil { + api.WriteError(w, err.Error(), http.StatusInternalServerError) + } else { + security.DecorateSessionTokenResponse(token, sessionUser) + api.WriteOKJSON(w, token) + } +} + +func buildRefreshedSession(reqUser *security.UserSessionInfo) (*security.UserSessionInfo, error) { + if reqUser == nil { + return nil, fmt.Errorf("user not found") + } + + sessionUser := cloneSessionUser(reqUser) + if reqUser.Provider != security.DefaultNativeAuthBackend { + return sessionUser, nil + } + + provider, account, err := security.GetUserByID(reqUser.UserID) + if err != nil { + return nil, err + } + if account == nil { + return nil, fmt.Errorf("user not found") + } + + login := strings.TrimSpace(account.Email) + if login == "" { + login = strings.TrimSpace(reqUser.Login) + } + if provider == "" { + provider = security.DefaultNativeAuthBackend + } + + sessionUser = &security.UserSessionInfo{ + Provider: provider, + Login: login, + Roles: append([]string(nil), account.Roles...), + Permissions: append([]security.PermissionKey(nil), reqUser.Permissions...), + LastLogin: reqUser.LastLogin, + } + sessionUser.SetUserID(account.ID) + return sessionUser, nil +} + +func cloneSessionUser(reqUser *security.UserSessionInfo) *security.UserSessionInfo { + if reqUser == nil { + return nil + } + + sessionUser := &security.UserSessionInfo{ + Provider: reqUser.Provider, + Login: reqUser.Login, + Roles: append([]string(nil), reqUser.Roles...), + Permissions: append([]security.PermissionKey(nil), reqUser.Permissions...), + LastLogin: reqUser.LastLogin, + } + sessionUser.SetUserID(reqUser.UserID) + return sessionUser +} diff --git a/modules/security/account/refresh_test.go b/modules/security/account/refresh_test.go new file mode 100644 index 000000000..41e49aecb --- /dev/null +++ b/modules/security/account/refresh_test.go @@ -0,0 +1,98 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package account + +import ( + "testing" + + "infini.sh/framework/core/security" +) + +type refreshTestProvider struct{} + +func (refreshTestProvider) GetUserByID(id string) (bool, *security.UserAccount, error) { + if id != "refresh-native-user" { + return false, nil, nil + } + + account := &security.UserAccount{ + Name: "Refreshed Admin", + Email: "refreshed@example.org", + Roles: []string{security.RoleAdmin}, + } + account.ID = id + return true, account, nil +} + +func (refreshTestProvider) GetUserByLogin(login string) (bool, *security.UserAccount, error) { + return false, nil, nil +} + +func (refreshTestProvider) CreateUser(name, login, password string, force bool) (*security.UserAccount, error) { + return nil, nil +} + +// External providers can keep their current session payload when refreshing. +func TestBuildRefreshedSessionKeepsExternalUser(t *testing.T) { + reqUser := &security.UserSessionInfo{ + Provider: "sso", + Login: "alice@example.org", + Roles: []string{"viewer"}, + } + reqUser.SetUserID("external-1") + + sessionUser, err := buildRefreshedSession(reqUser) + if err != nil { + t.Fatalf("build refreshed session: %v", err) + } + if sessionUser.Login != reqUser.Login { + t.Fatalf("expected external login %q, got %q", reqUser.Login, sessionUser.Login) + } + if sessionUser.UserID != reqUser.UserID { + t.Fatalf("expected external user id %q, got %q", reqUser.UserID, sessionUser.UserID) + } +} + +// Native refreshes should pull the latest account snapshot from the registered backend. +func TestBuildRefreshedSessionReloadsNativeAccount(t *testing.T) { + security.RegisterAuthenticationProvider("refresh-test-provider", refreshTestProvider{}) + + reqUser := &security.UserSessionInfo{ + Provider: security.DefaultNativeAuthBackend, + Login: "stale@example.org", + Roles: []string{"viewer"}, + } + reqUser.SetUserID("refresh-native-user") + + sessionUser, err := buildRefreshedSession(reqUser) + if err != nil { + t.Fatalf("build refreshed session: %v", err) + } + if sessionUser.Login != "refreshed@example.org" { + t.Fatalf("expected refreshed login, got %q", sessionUser.Login) + } + if len(sessionUser.Roles) != 1 || sessionUser.Roles[0] != security.RoleAdmin { + t.Fatalf("expected refreshed roles, got %#v", sessionUser.Roles) + } +} diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go index 1b0672800..eba0ce312 100644 --- a/modules/security/rbac/account_login.go +++ b/modules/security/rbac/account_login.go @@ -145,23 +145,23 @@ func Login(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { return } + usedChallenge := req.ChallengeID != "" || req.Proof != "" exists, user, err := lookupAccountByLogin(login) if err != nil { api.WriteError(w, err.Error(), http.StatusInternalServerError) return } - if !exists || user == nil { + if usedChallenge && (!exists || user == nil) { api.WriteError(w, errInvalidLoginCredentials.Error(), http.StatusForbidden) return } - usedChallenge := req.ChallengeID != "" || req.Proof != "" if err := validateReplayNonce(r, usedChallenge); err != nil { api.WriteError(w, err.Error(), http.StatusUnauthorized) return } - usedChallenge, err = authenticateLogin(user, login, req.Password, req.ChallengeID, req.Proof) + usedChallenge, sessionUser, nativeUser, err := authenticateLogin(user, login, req.Password, req.ChallengeID, req.Proof) if err != nil { statusCode := http.StatusForbidden if errors.Is(err, errIncompleteChallenge) || errors.Is(err, errMissingPassword) { @@ -171,14 +171,14 @@ func Login(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { return } - if !usedChallenge { - upgradePasswordChallenge(user, req.Password) + if !usedChallenge && nativeUser != nil { + upgradePasswordChallenge(nativeUser, req.Password) } - sessionUser := newNativeSession(user, login) if err, token := security.AddUserToSession(w, r, sessionUser); err != nil { api.WriteError(w, err.Error(), http.StatusInternalServerError) } else { + security.DecorateSessionTokenResponse(token, sessionUser) api.WriteOKJSON(w, token) } } @@ -218,33 +218,44 @@ func buildLoginChallengeResponse(login string, exists bool, user *security.UserA } // authenticateLogin selects the correct credential validation path based on the request body. -func authenticateLogin(user *security.UserAccount, login, password, challengeID, proof string) (bool, error) { - if user == nil { - return false, errInvalidLoginCredentials - } - +func authenticateLogin(user *security.UserAccount, login, password, challengeID, proof string) (bool, *security.UserSessionInfo, *security.UserAccount, error) { if challengeID != "" || proof != "" { if challengeID == "" || proof == "" { - return true, errIncompleteChallenge + return true, nil, nil, errIncompleteChallenge } + if user == nil { + return true, nil, nil, errInvalidLoginCredentials + } challenge, err := security.ConsumeLoginChallenge(challengeID, login) if err != nil || !security.CanUsePasswordChallenge(user) { - return true, errInvalidLoginCredentials + return true, nil, nil, errInvalidLoginCredentials } if !security.VerifyPasswordProof(user.PasswordVerifier, login, challenge.ID, challenge.Nonce, proof) { - return true, errInvalidLoginCredentials + return true, nil, nil, errInvalidLoginCredentials } - return true, nil + return true, newNativeSession(user, login), user, nil } if password == "" { - return false, errMissingPassword + return false, nil, nil, errMissingPassword + } + + if user != nil { + if err := security.VerifyPassword(user, password); err == nil { + return false, newNativeSession(user, login), user, nil + } + } + + sessionUser, err := security.AuthenticateAccountPasswordLogin(login, password) + if err != nil { + return false, nil, nil, err } - if err := security.VerifyPassword(user, password); err != nil { - return false, errInvalidLoginCredentials + if sessionUser != nil { + return false, sessionUser, nil, nil } - return false, nil + + return false, nil, nil, errInvalidLoginCredentials } // lookupAccountByLogin normalizes the service-registry "not found" result into a regular miss. diff --git a/modules/security/rbac/account_login_test.go b/modules/security/rbac/account_login_test.go index fe66941db..6b4b60a75 100644 --- a/modules/security/rbac/account_login_test.go +++ b/modules/security/rbac/account_login_test.go @@ -28,11 +28,28 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "infini.sh/framework/core/security" replaysecurity "infini.sh/framework/core/security/replay" ) +type testAccountPasswordLoginProvider struct{} + +func (testAccountPasswordLoginProvider) AuthenticateByPassword(login, password string) (*security.UserSessionInfo, error) { + if login != "ldap-user" || password != "StrongPassw0rd!" { + return nil, nil + } + + sessionUser := &security.UserSessionInfo{ + Provider: "ldap", + Login: login, + Roles: []string{"viewer"}, + } + sessionUser.SetUserID("ldap-user-id") + return sessionUser, nil +} + // The request payload accepts multiple historical login field names during rollout. func TestAccountLoginRequestNormalizedLogin(t *testing.T) { req := accountLoginRequest{ @@ -52,13 +69,16 @@ func TestAuthenticateLoginWithPassword(t *testing.T) { t.Fatalf("set password: %v", err) } - usedChallenge, err := authenticateLogin(user, user.Email, "StrongPassw0rd!", "", "") + usedChallenge, sessionUser, nativeUser, err := authenticateLogin(user, user.Email, "StrongPassw0rd!", "", "") if err != nil { t.Fatalf("authenticate login: %v", err) } if usedChallenge { t.Fatal("expected password login path") } + if sessionUser == nil || nativeUser == nil { + t.Fatalf("expected native password login state, got session=%#v native=%#v", sessionUser, nativeUser) + } } // Challenge login should succeed once the account already has verifier material. @@ -74,13 +94,16 @@ func TestAuthenticateLoginWithChallenge(t *testing.T) { t.Fatalf("build password proof: %v", err) } - usedChallenge, err := authenticateLogin(user, user.Email, "", challenge.ID, proof) + usedChallenge, sessionUser, nativeUser, err := authenticateLogin(user, user.Email, "", challenge.ID, proof) if err != nil { t.Fatalf("authenticate login: %v", err) } if !usedChallenge { t.Fatal("expected challenge login path") } + if sessionUser == nil || nativeUser == nil { + t.Fatalf("expected native challenge login state, got session=%#v native=%#v", sessionUser, nativeUser) + } } // Partially supplied challenge payloads should fail distinctly from bad credentials. @@ -90,7 +113,7 @@ func TestAuthenticateLoginRejectsIncompleteChallenge(t *testing.T) { t.Fatalf("set password: %v", err) } - _, err := authenticateLogin(user, user.Email, "", "challenge-id", "") + _, _, _, err := authenticateLogin(user, user.Email, "", "challenge-id", "") if !errors.Is(err, errIncompleteChallenge) { t.Fatalf("expected incomplete challenge error, got %v", err) } @@ -104,12 +127,31 @@ func TestAuthenticateLoginRejectsWrongProof(t *testing.T) { } challenge := security.NewLoginChallenge(user.Email) - _, err := authenticateLogin(user, user.Email, "", challenge.ID, "bad-proof") + _, _, _, err := authenticateLogin(user, user.Email, "", challenge.ID, "bad-proof") if !errors.Is(err, errInvalidLoginCredentials) { t.Fatalf("expected invalid credential error, got %v", err) } } +// Applications can attach non-native password realms to the shared framework login flow. +func TestAuthenticateLoginFallsBackToRegisteredPasswordProvider(t *testing.T) { + security.RegisterAccountPasswordLoginProvider("test-account-login", testAccountPasswordLoginProvider{}) + + usedChallenge, sessionUser, nativeUser, err := authenticateLogin(nil, "ldap-user", "StrongPassw0rd!", "", "") + if err != nil { + t.Fatalf("authenticate login: %v", err) + } + if usedChallenge { + t.Fatal("expected password fallback path") + } + if nativeUser != nil { + t.Fatalf("expected no native user for fallback path, got %#v", nativeUser) + } + if sessionUser == nil || sessionUser.Provider != "ldap" { + t.Fatalf("expected ldap session user, got %#v", sessionUser) + } +} + // Older accounts intentionally advertise plain login until their verifier is available. func TestBuildLoginChallengeResponseFallsBackToPlain(t *testing.T) { user := &security.UserAccount{Email: "admin@example.org"} @@ -188,3 +230,38 @@ func TestNewNativeSessionFallsBackToRequestedLogin(t *testing.T) { t.Fatalf("expected native provider, got %q", session.Provider) } } + +// The framework login response keeps the console frontend contract while the handler +// implementation moves from console into framework-owned routes. +func TestDecorateLoginResponseAddsConsoleCompatibilityFields(t *testing.T) { + session := &security.UserSessionInfo{ + Provider: security.DefaultNativeAuthBackend, + Login: "admin@example.org", + Roles: []string{security.RoleAdmin}, + Permissions: []security.PermissionKey{security.GetSimplePermission("generic", "unit", security.Read)}, + } + session.SetUserID("user-1") + + token := map[string]interface{}{ + "status": "ok", + "expire_in": time.Now().Unix() + 3600, + } + security.DecorateSessionTokenResponse(token, session) + + if token["username"] != session.Login { + t.Fatalf("expected username %q, got %v", session.Login, token["username"]) + } + if token["id"] != session.UserID { + t.Fatalf("expected id %q, got %v", session.UserID, token["id"]) + } + if token["expires_at"] == nil { + t.Fatal("expected expires_at to be populated") + } + if expireIn, ok := token["expire_in"].(int64); !ok || expireIn <= 0 || expireIn > 3600 { + t.Fatalf("expected expire_in to become remaining lifetime seconds, got %#v", token["expire_in"]) + } + privilege, ok := token["privilege"].([]security.PermissionKey) + if !ok || len(privilege) == 0 { + t.Fatalf("expected privilege list to be populated, got %#v", token["privilege"]) + } +} From f595d1728325c7fef7c7bf610757951ded3222c4 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 26 May 2026 17:18:24 +0800 Subject: [PATCH 053/137] Avoid panic in RBAC user flow Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- modules/security/rbac/user.go | 84 ++++++++++++++++++++---------- modules/security/rbac/user_test.go | 38 ++++++++++++++ 2 files changed, 95 insertions(+), 27 deletions(-) create mode 100644 modules/security/rbac/user_test.go diff --git a/modules/security/rbac/user.go b/modules/security/rbac/user.go index 64d36b44e..f8ea5c6a2 100644 --- a/modules/security/rbac/user.go +++ b/modules/security/rbac/user.go @@ -11,12 +11,20 @@ import ( "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" + cerr "infini.sh/framework/core/errors" "infini.sh/framework/core/global" "infini.sh/framework/core/orm" "infini.sh/framework/core/security" "infini.sh/framework/core/util" ) +const ( + errCannotUpdateOwnRoles = "sorry, you can not update your roles" + errCannotDeleteSelf = "you can not delete yourself" + errInsecurePassword = "password does not meet security requirements" + errEmailAlreadyExists = "email already existed" +) + func GetUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") @@ -25,10 +33,11 @@ func GetUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { ctx := orm.NewContextWithParent(req.Context()) exists, err := orm.GetV2(ctx, &obj) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } if !exists { - api.NotFoundResponse(id) + api.WriteJSON(w, api.NotFoundResponse(id), http.StatusNotFound) return } @@ -42,7 +51,8 @@ func UpdateUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) obj := security.UserAccount{} err := api.DecodeJSON(req, &obj) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusBadRequest) + return } api.MustValidateInput(w, obj) @@ -51,10 +61,11 @@ func UpdateUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) oldObj.ID = id exists, err := orm.GetV2(ctx, &oldObj) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } if !exists { - api.NotFoundResponse(id) + api.WriteJSON(w, api.NotFoundResponse(id), http.StatusNotFound) return } @@ -67,7 +78,8 @@ func UpdateUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) if userID == id { //user can't update self's role if !util.CompareStringArray(obj.Roles, oldObj.Roles) { - panic("sorry, you can not update your roles") + api.WriteError(w, errCannotUpdateOwnRoles, http.StatusForbidden) + return } } @@ -78,17 +90,20 @@ func UpdateUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) obj.PasswordSalt = oldObj.PasswordSalt obj.PasswordVerifier = oldObj.PasswordVerifier } else { - if !util.ValidateSecure(obj.Password) { - panic("should be secured password") + if err := validateSecurePassword(obj.Password); err != nil { + api.WriteError(w, err.Error(), http.StatusBadRequest) + return } if err := security.SetPassword(&obj, obj.Password); err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } } ctx.Refresh = orm.WaitForRefresh err = orm.Update(ctx, &obj) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } security.IncreasePermissionVersion() @@ -104,13 +119,15 @@ func DeleteUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) sessionUser := security.MustGetUserFromContext(ctx) userID := sessionUser.MustGetUserID() if userID == id { - panic("you can not delete yourself") + api.WriteError(w, errCannotDeleteSelf, http.StatusForbidden) + return } ctx.Refresh = orm.WaitForRefresh err := orm.Delete(ctx, &obj) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } api.WriteDeletedOKJSON(w, obj.ID) @@ -119,7 +136,8 @@ func DeleteUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) func SearchUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { builder, err := orm.NewQueryBuilderFromRequest(req, "id", "name", "email") if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusBadRequest) + return } ctx := orm.NewContextWithParent(req.Context()) ctx.DirectReadAccess() @@ -129,12 +147,13 @@ func SearchUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) orm.WithModel(ctx, &security.UserAccount{}) res, err := orm.SearchV2(ctx, builder) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } _, err = api.Write(w, res.Payload.([]byte)) if err != nil { - panic(err) + api.Error(w, err) } } @@ -187,17 +206,17 @@ func (provider *SecurityBackendProvider) GetUserByID(id string) (bool, *security func (provider *SecurityBackendProvider) CreateUser(name, email, password string, force bool) (*security.UserAccount, error) { - if !util.ValidateSecure(password) { - panic("should be secured password") + if err := validateSecurePassword(password); err != nil { + return nil, err } exists, account, err := GetUserByLogin(email) if err != nil { - panic(err) + return nil, err } if exists && !force { - panic("email already existed") + return nil, cerr.NewWithHTTPCode(http.StatusConflict, errEmailAlreadyExists) } var obj = &security.UserAccount{} @@ -212,7 +231,7 @@ func (provider *SecurityBackendProvider) CreateUser(name, email, password string obj.Email = email obj.Roles = []string{security.RoleAdmin} if err := security.SetPassword(obj, password); err != nil { - panic(err) + return nil, err } ctx := orm.NewContext() @@ -220,7 +239,7 @@ func (provider *SecurityBackendProvider) CreateUser(name, email, password string ctx.Refresh = orm.WaitForRefresh err = orm.Save(ctx, obj) if err != nil { - panic(err) + return nil, err } return obj, nil } @@ -233,33 +252,37 @@ func CreateUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) var obj = &security.UserAccount{} err := api.DecodeJSON(req, obj) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusBadRequest) + return } api.MustValidateInput(w, obj) exists, account, err := GetUserByLogin(obj.Email) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } if exists && account != nil { log.Warn("email already exists") - //obj.ID = account.ID - panic("email already existed") + api.WriteError(w, errEmailAlreadyExists, http.StatusConflict) + return } else { obj.ID = getUIDByEmail(obj.Email) } randStr := util.GenerateSecureString(8) if err := security.SetPassword(obj, randStr); err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } ctx := orm.NewContextWithParent(req.Context()) ctx.Refresh = orm.WaitForRefresh err = orm.Save(ctx, obj) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } obj.Password = randStr @@ -269,3 +292,10 @@ func CreateUser(w http.ResponseWriter, req *http.Request, ps httprouter.Params) obj.PasswordVerifier = "" api.WriteJSON(w, obj, 200) } + +func validateSecurePassword(password string) error { + if util.ValidateSecure(password) { + return nil + } + return cerr.NewWithHTTPCode(http.StatusBadRequest, errInsecurePassword) +} diff --git a/modules/security/rbac/user_test.go b/modules/security/rbac/user_test.go new file mode 100644 index 000000000..be4a9333f --- /dev/null +++ b/modules/security/rbac/user_test.go @@ -0,0 +1,38 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package rbac + +import "testing" + +// Weak passwords should now fail as normal validation errors instead of aborting +// the request flow via panic. +func TestValidateSecurePassword(t *testing.T) { + if err := validateSecurePassword("weak"); err == nil { + t.Fatal("expected weak password to be rejected") + } + + if err := validateSecurePassword("StrongPassw0rd!"); err != nil { + t.Fatalf("expected strong password to pass validation, got %v", err) + } +} From be86a28fb91eb3910f80340f0e8392858c0c97d0 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 26 May 2026 17:31:53 +0800 Subject: [PATCH 054/137] Clean auth runtime panics Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- core/security/session.go | 2 +- core/security/session_test.go | 36 ++++++++++++++++++++++++ core/security/user_session.go | 4 +-- core/security/user_session_test.go | 11 ++++++++ core/security/validate_test.go | 27 ++++++++++++++++++ modules/security/rbac/entity.go | 4 ++- modules/security/rbac/principal.go | 6 ++-- modules/security/rbac/role.go | 45 ++++++++++++++++++++++-------- 8 files changed, 118 insertions(+), 17 deletions(-) create mode 100644 core/security/session_test.go diff --git a/core/security/session.go b/core/security/session.go index 6f52e016e..789cc9230 100644 --- a/core/security/session.go +++ b/core/security/session.go @@ -66,7 +66,7 @@ func byAccessTokenSession(w http.ResponseWriter, r *http.Request) (claims *UserC func AddUserToSession(w http.ResponseWriter, r *http.Request, user *UserSessionInfo) (error, map[string]interface{}) { if user == nil { - panic("invalid user") + return errors.NewWithHTTPCode(http.StatusUnauthorized, "invalid user"), nil } // Generate access token diff --git a/core/security/session_test.go b/core/security/session_test.go new file mode 100644 index 000000000..aaa1d5cba --- /dev/null +++ b/core/security/session_test.go @@ -0,0 +1,36 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package security + +import "testing" + +func TestAddUserToSessionRejectsNilUser(t *testing.T) { + err, token := AddUserToSession(nil, nil, nil) + if err == nil { + t.Fatal("expected nil user to be rejected") + } + if token != nil { + t.Fatalf("expected no token for nil user, got %+v", token) + } +} diff --git a/core/security/user_session.go b/core/security/user_session.go index 0bbcddfb8..5451435f1 100644 --- a/core/security/user_session.go +++ b/core/security/user_session.go @@ -147,7 +147,7 @@ func (u *UserSessionInfo) MustGetUserID() string { return u.UserID } - panic(errors.NewWithHTTPCode(400, "invalid user")) + panic(errors.NewWithHTTPCode(401, "invalid user")) } func (u *UserSessionInfo) IsValid() bool { @@ -156,7 +156,7 @@ func (u *UserSessionInfo) IsValid() bool { if global.Env().IsDebug { log.Error(util.MustToJSON(u), u.UserID) } - panic(errors.NewWithHTTPCode(400, "invalid user")) + return false } return v } diff --git a/core/security/user_session_test.go b/core/security/user_session_test.go index 41b37eb19..d383b829a 100644 --- a/core/security/user_session_test.go +++ b/core/security/user_session_test.go @@ -86,3 +86,14 @@ func TestUserClaimsUnmarshalAcceptsLegacyConsoleAliases(t *testing.T) { t.Fatalf("expected provider to be preserved, got %q", claims.Provider) } } + +func TestUserSessionInfoIsValidReturnsFalseForIncompleteUser(t *testing.T) { + user := &UserSessionInfo{ + Provider: "native", + Login: "admin@example.org", + } + + if user.IsValid() { + t.Fatal("expected incomplete user session to be invalid") + } +} diff --git a/core/security/validate_test.go b/core/security/validate_test.go index 144a6f8c5..6ad896983 100644 --- a/core/security/validate_test.go +++ b/core/security/validate_test.go @@ -66,3 +66,30 @@ func TestValidateAuthorizationHeader(t *testing.T) { t.Fatalf("expected user id to round-trip, got %q", sessionUser.UserID) } } + +func TestValidateAuthorizationHeaderRejectsIncompleteUserClaims(t *testing.T) { + oldSecret := secretKey + secretKey = "test-framework-secret" + defer func() { + secretKey = oldSecret + }() + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, UserClaims{ + RegisteredClaims: &jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour)), + }, + UserSessionInfo: &UserSessionInfo{ + Provider: "native", + Login: "admin@example.org", + }, + }) + tokenString, err := token.SignedString([]byte(secretKey)) + if err != nil { + t.Fatalf("sign token: %v", err) + } + + sessionUser, err := ValidateAuthorizationHeader("Bearer " + tokenString) + if err == nil { + t.Fatalf("expected invalid claims to be rejected, got user %+v", sessionUser) + } +} diff --git a/modules/security/rbac/entity.go b/modules/security/rbac/entity.go index 20544681d..9a839766f 100644 --- a/modules/security/rbac/entity.go +++ b/modules/security/rbac/entity.go @@ -7,6 +7,7 @@ package rbac import ( "context" + log "github.com/cihub/seelog" "infini.sh/framework/core/elastic" "infini.sh/framework/core/entity_card" "infini.sh/framework/core/orm" @@ -47,7 +48,8 @@ func (this *UserEntityProvider) GenEntityLabel(ctx1 context.Context, t string, i out := []security.UserAccount{} err, _ := elastic.SearchV2WithResultItemMapper(ctx, &out, builder, nil) if err != nil { - panic(err) + log.Errorf("failed to load user entity labels for ids %v: %v", ids, err) + return output } for _, a := range out { diff --git a/modules/security/rbac/principal.go b/modules/security/rbac/principal.go index 1f24d27a8..d21d91a61 100644 --- a/modules/security/rbac/principal.go +++ b/modules/security/rbac/principal.go @@ -18,7 +18,8 @@ func SearchPrincipals(w http.ResponseWriter, req *http.Request, ps httprouter.Pa builder, err := orm.NewQueryBuilderFromRequest(req, "id", "name", "email") if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusBadRequest) + return } ctx := orm.NewContextWithParent(req.Context()) ctx.DirectReadAccess() @@ -26,7 +27,8 @@ func SearchPrincipals(w http.ResponseWriter, req *http.Request, ps httprouter.Pa out := []security.UserAccount{} err, res := elastic.SearchV2WithResultItemMapper(ctx, &out, builder, nil) if err != nil { - panic(err) + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return } // use the generic type correctly diff --git a/modules/security/rbac/role.go b/modules/security/rbac/role.go index 2bda923ad..92c175b37 100644 --- a/modules/security/rbac/role.go +++ b/modules/security/rbac/role.go @@ -8,6 +8,7 @@ import ( "context" "net/http" + log "github.com/cihub/seelog" "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/elastic" @@ -17,6 +18,14 @@ import ( "infini.sh/framework/core/util" ) +const ( + errInvalidCurrentUser = "invalid user" + errInvalidRole = "invalid role" + errCannotUpdateOwnRole = "you can not update the roles for you" + errReservedRoleName = "can not use the reserved role name" + errRoleAlreadyExists = "same role name already exists" +) + func GetRole(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { id := ps.MustGetParameter("id") @@ -28,8 +37,12 @@ func GetRole(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { ctx.PermissionScope(security.PermissionScopePlatform) exists, err := orm.GetV2(ctx, &obj) - if !exists || err != nil { - api.NotFoundResponse(id) + if err != nil { + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !exists { + api.WriteJSON(w, api.NotFoundResponse(id), http.StatusNotFound) return } @@ -44,7 +57,7 @@ func UpdateRole(w http.ResponseWriter, req *http.Request, ps httprouter.Params) obj := security.UserRole{} err := api.DecodeJSON(req, &obj) if err != nil { - api.WriteError(w, err.Error(), http.StatusInternalServerError) + api.WriteError(w, err.Error(), http.StatusBadRequest) return } @@ -56,16 +69,23 @@ func UpdateRole(w http.ResponseWriter, req *http.Request, ps httprouter.Params) userID := sessionUser.MustGetUserID() _, account, err := security.GetUserByID(userID) - if account == nil || err != nil { - panic("invalid user") + if err != nil { + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if account == nil { + api.WriteError(w, errInvalidCurrentUser, http.StatusUnauthorized) + return } _, role := GetRoleByID(id) if role == nil { - panic("invalid role") + api.WriteError(w, errInvalidRole, http.StatusNotFound) + return } if util.ContainsAnyInArray(role.Name, account.Roles) { - panic("you can not update the roles for you") + api.WriteError(w, errCannotUpdateOwnRole, http.StatusForbidden) + return } } @@ -164,19 +184,21 @@ func CreateRole(w http.ResponseWriter, req *http.Request, ps httprouter.Params) var obj = &security.UserRole{} err := api.DecodeJSON(req, obj) if err != nil { - api.WriteError(w, err.Error(), http.StatusInternalServerError) + api.WriteError(w, err.Error(), http.StatusBadRequest) return } if obj.Name == "admin" { - panic("can not use the reserved role name") + api.WriteError(w, errReservedRoleName, http.StatusBadRequest) + return } api.MustValidateInput(w, obj) exists, _ := GetRoleByName(obj.Name) if exists { - panic("same role name already exists") + api.WriteError(w, errRoleAlreadyExists, http.StatusConflict) + return } ctx := orm.NewContextWithParent(req.Context()) @@ -237,7 +259,8 @@ func (provider *SecurityBackendProvider) GetPermissionKeysByRoles(ctx1 context.C result := []security.UserRole{} err, _ := elastic.SearchV2WithResultItemMapper(ctx, &result, qb, nil) if err != nil { - panic(err) + log.Errorf("failed to load permissions for roles %v: %v", roles, err) + return []security.PermissionKey{} } allowed := make(map[security.PermissionKey]struct{}, 128) From 62421be1c8926ce6ca922a819cc8a21bceb8c72b Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 26 May 2026 17:45:49 +0800 Subject: [PATCH 055/137] Inline RBAC account routes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- modules/security/rbac/account_login.go | 22 ---------------------- modules/security/rbac/init.go | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go index eba0ce312..e3e813f86 100644 --- a/modules/security/rbac/account_login.go +++ b/modules/security/rbac/account_login.go @@ -59,28 +59,6 @@ type accountLoginRequest struct { Proof string `json:"proof"` } -func registerAccountRoutes() { - // These endpoints are only registered from rbac.Init(), so they exist only when - // native authentication is enabled and the native user backend is ready. - api.HandleUIMethod(api.POST, "/account/replay_nonce", - api.RequireSecureTransport(IssueReplayNonce), - api.AllowPublicAccess(), - api.AllowOPTIONSS(), - api.Feature(api.FeatureCORS)) - - api.HandleUIMethod(api.POST, "/account/login/challenge", - api.RequireSecureTransport(LoginChallenge), - api.AllowPublicAccess(), - api.AllowOPTIONSS(), - api.Feature(api.FeatureCORS)) - - api.HandleUIMethod(api.POST, "/account/login", - api.RequireSecureTransport(Login), - api.AllowPublicAccess(), - api.AllowOPTIONSS(), - api.Feature(api.FeatureCORS)) -} - // IssueReplayNonce mints a short-lived nonce bound to the caller and target request scope. func IssueReplayNonce(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { var req struct { diff --git a/modules/security/rbac/init.go b/modules/security/rbac/init.go index 1a2a022ab..3b6290421 100644 --- a/modules/security/rbac/init.go +++ b/modules/security/rbac/init.go @@ -18,7 +18,24 @@ func Init() { provider := SecurityBackendProvider{} security.RegisterAuthenticationProvider(security.DefaultNativeAuthBackend, &provider) security.RegisterAuthorizationProvider(security.DefaultNativeAuthBackend, &provider) - registerAccountRoutes() + + api.HandleUIMethod(api.POST, "/account/replay_nonce", + api.RequireSecureTransport(IssueReplayNonce), + api.AllowPublicAccess(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS)) + + api.HandleUIMethod(api.POST, "/account/login/challenge", + api.RequireSecureTransport(LoginChallenge), + api.AllowPublicAccess(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS)) + + api.HandleUIMethod(api.POST, "/account/login", + api.RequireSecureTransport(Login), + api.AllowPublicAccess(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS)) orm.MustRegisterSchemaWithIndexName(&security.UserAccount{}, "app-users") orm.MustRegisterSchemaWithIndexName(&security.UserRole{}, "app-roles") From 76d005b0ae6792512ca95ece2240afd08df3b59f Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 26 May 2026 17:49:36 +0800 Subject: [PATCH 056/137] Rehome account hook helpers Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- core/security/account_flow.go | 88 ------------------------------- core/security/service_registry.go | 42 +++++++++++++++ core/security/session.go | 21 ++++++++ 3 files changed, 63 insertions(+), 88 deletions(-) delete mode 100644 core/security/account_flow.go diff --git a/core/security/account_flow.go b/core/security/account_flow.go deleted file mode 100644 index 9cf0e886c..000000000 --- a/core/security/account_flow.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (C) INFINI Labs & INFINI LIMITED. -// -// The INFINI Framework is offered under the GNU Affero General Public License v3.0 -// and as commercial software. -// -// For commercial licensing, contact us at: -// - Website: infinilabs.com -// - Email: hello@infini.ltd -// -// Open Source licensed under AGPL V3: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package security - -import "sync" - -// AccountPasswordLoginProvider lets applications keep their own password-auth realms -// while reusing the framework-owned /account/login HTTP flow and session issuance. -type AccountPasswordLoginProvider interface { - AuthenticateByPassword(login, password string) (*UserSessionInfo, error) -} - -var accountPasswordLoginProviders = sync.Map{} - -func RegisterAccountPasswordLoginProvider(name string, provider AccountPasswordLoginProvider) { - accountPasswordLoginProviders.Store(name, provider) -} - -// AuthenticateAccountPasswordLogin tries application-provided password login providers -// after the native framework account path has either not matched or not succeeded. -func AuthenticateAccountPasswordLogin(login, password string) (*UserSessionInfo, error) { - var out *UserSessionInfo - var lastErr error - - accountPasswordLoginProviders.Range(func(key, value any) bool { - provider, ok := value.(AccountPasswordLoginProvider) - if !ok { - return true - } - - sessionUser, err := provider.AuthenticateByPassword(login, password) - if err != nil { - lastErr = err - return true - } - if sessionUser != nil { - out = sessionUser - return false - } - return true - }) - - if out != nil { - return out, nil - } - return nil, lastErr -} - -// SessionTokenResponseDecorator lets applications enrich the shared login/refresh -// response with app-specific fields while reusing the framework session pipeline. -type SessionTokenResponseDecorator func(token map[string]interface{}, user *UserSessionInfo) - -var sessionTokenResponseDecorators = sync.Map{} - -func RegisterSessionTokenResponseDecorator(name string, decorator SessionTokenResponseDecorator) { - sessionTokenResponseDecorators.Store(name, decorator) -} - -func applySessionTokenResponseDecorators(token map[string]interface{}, user *UserSessionInfo) { - sessionTokenResponseDecorators.Range(func(key, value any) bool { - decorator, ok := value.(SessionTokenResponseDecorator) - if ok { - decorator(token, user) - } - return true - }) -} diff --git a/core/security/service_registry.go b/core/security/service_registry.go index 6f375906b..494c4ec3f 100644 --- a/core/security/service_registry.go +++ b/core/security/service_registry.go @@ -21,6 +21,12 @@ type AuthorizationBackend interface { GetPermissionKeysByRoles(ctx context.Context, roles []string) []PermissionKey } +// AccountPasswordLoginProvider lets applications keep their own password-auth realms +// while reusing the shared framework account login handler and session issuance. +type AccountPasswordLoginProvider interface { + AuthenticateByPassword(login, password string) (*UserSessionInfo, error) +} + var authorizationBackendProviders = sync.Map{} func RegisterAuthorizationProvider(name string, provider AuthorizationBackend) { @@ -33,6 +39,12 @@ func RegisterAuthenticationProvider(name string, provider AuthenticationBackend) authenticationBackendBackendProviders.Store(name, provider) } +var accountPasswordLoginProviders = sync.Map{} + +func RegisterAccountPasswordLoginProvider(name string, provider AccountPasswordLoginProvider) { + accountPasswordLoginProviders.Store(name, provider) +} + func MustGetAuthenticationProvider(provider string) AuthenticationBackend { value, ok := authenticationBackendBackendProviders.Load(provider) if ok { @@ -99,3 +111,33 @@ func GetUserByLogin(login string) (bool, *UserAccount, error) { return false, nil, errors.New("not found") } + +// AuthenticateAccountPasswordLogin tries application-provided password login providers +// after the native framework account path has either not matched or not succeeded. +func AuthenticateAccountPasswordLogin(login, password string) (*UserSessionInfo, error) { + var out *UserSessionInfo + var lastErr error + + accountPasswordLoginProviders.Range(func(key, value any) bool { + provider, ok := value.(AccountPasswordLoginProvider) + if !ok { + return true + } + + sessionUser, err := provider.AuthenticateByPassword(login, password) + if err != nil { + lastErr = err + return true + } + if sessionUser != nil { + out = sessionUser + return false + } + return true + }) + + if out != nil { + return out, nil + } + return nil, lastErr +} diff --git a/core/security/session.go b/core/security/session.go index 789cc9230..abb75a3db 100644 --- a/core/security/session.go +++ b/core/security/session.go @@ -7,6 +7,7 @@ package security import ( "fmt" "net/http" + "sync" "time" "github.com/golang-jwt/jwt/v4" @@ -18,10 +19,20 @@ import ( const UserAccessTokenSessionName = "user_session_access_token" const UserAccessTokenTTL = 24 * time.Hour +// SessionTokenResponseDecorator lets applications enrich the shared login/refresh +// response with app-specific fields while reusing the framework session pipeline. +type SessionTokenResponseDecorator func(token map[string]interface{}, user *UserSessionInfo) + +var sessionTokenResponseDecorators = sync.Map{} + func init() { RegisterHTTPAuthFilterProvider("session_token", byAccessTokenSession) } +func RegisterSessionTokenResponseDecorator(name string, decorator SessionTokenResponseDecorator) { + sessionTokenResponseDecorators.Store(name, decorator) +} + func byAccessTokenSession(w http.ResponseWriter, r *http.Request) (claims *UserClaims, err error) { exists, sessToken := api.GetSession(w, r, UserAccessTokenSessionName) if !exists || sessToken == nil { @@ -155,3 +166,13 @@ func tokenExpiresAtUnix(value interface{}) int64 { return 0 } } + +func applySessionTokenResponseDecorators(token map[string]interface{}, user *UserSessionInfo) { + sessionTokenResponseDecorators.Range(func(key, value any) bool { + decorator, ok := value.(SessionTokenResponseDecorator) + if ok { + decorator(token, user) + } + return true + }) +} From 28d23e3a635cb4716f725a979ca2c083933f1372 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 26 May 2026 17:59:02 +0800 Subject: [PATCH 057/137] Tighten account login semantics Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- modules/security/rbac/account_login.go | 16 ++++------------ modules/security/rbac/user.go | 23 +++++++++++++++-------- modules/security/rbac/user_test.go | 26 +++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go index e3e813f86..2e630fb97 100644 --- a/modules/security/rbac/account_login.go +++ b/modules/security/rbac/account_login.go @@ -99,7 +99,7 @@ func LoginChallenge(w http.ResponseWriter, r *http.Request, ps httprouter.Params return } - exists, user, err := lookupAccountByLogin(login) + exists, user, err := GetUserByLogin(login) if err != nil { api.WriteError(w, err.Error(), http.StatusInternalServerError) return @@ -124,7 +124,7 @@ func Login(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { } usedChallenge := req.ChallengeID != "" || req.Proof != "" - exists, user, err := lookupAccountByLogin(login) + exists, user, err := GetUserByLogin(login) if err != nil { api.WriteError(w, err.Error(), http.StatusInternalServerError) return @@ -236,15 +236,6 @@ func authenticateLogin(user *security.UserAccount, login, password, challengeID, return false, nil, nil, errInvalidLoginCredentials } -// lookupAccountByLogin normalizes the service-registry "not found" result into a regular miss. -func lookupAccountByLogin(login string) (bool, *security.UserAccount, error) { - exists, user, err := GetUserByLogin(login) - if err != nil && err.Error() == "not found" { - return false, nil, nil - } - return exists, user, err -} - // validateReplayNonce keeps challenge login replay-safe while leaving older password-only // clients working until they adopt the explicit nonce negotiation endpoint. func validateReplayNonce(r *http.Request, required bool) error { @@ -271,9 +262,10 @@ func upgradePasswordChallenge(user *security.UserAccount, password string) { // Persist the verifier after a successful legacy password login so subsequent // logins can move onto the challenge flow without an explicit migration step. + // This upgrade is best-effort; the current login already succeeded, so it should + // not wait for an index refresh before returning to the caller. ctx := orm.NewContext() ctx.DirectAccess() - ctx.Refresh = orm.WaitForRefresh if err := orm.Update(ctx, user); err != nil { log.Warnf("failed to persist password challenge for user [%s]: %v", user.Email, err) } diff --git a/modules/security/rbac/user.go b/modules/security/rbac/user.go index f8ea5c6a2..018f58f3c 100644 --- a/modules/security/rbac/user.go +++ b/modules/security/rbac/user.go @@ -5,6 +5,7 @@ package rbac import ( + "fmt" "net/http" log "github.com/cihub/seelog" @@ -172,14 +173,8 @@ func GetUserByLogin(email string) (bool, *security.UserAccount, error) { if err != nil { return false, nil, err } - if len(items) > 0 { - if len(items) == 1 { - return true, &items[0], nil - } else { - log.Warnf("invalid users, more than one account was associated with the same email: %v", email) - } - } - return false, nil, nil + + return resolveUserByLogin(email, items) } func (provider *SecurityBackendProvider) GetUserByLogin(email string) (bool, *security.UserAccount, error) { @@ -299,3 +294,15 @@ func validateSecurePassword(password string) error { } return cerr.NewWithHTTPCode(http.StatusBadRequest, errInsecurePassword) } + +func resolveUserByLogin(login string, items []security.UserAccount) (bool, *security.UserAccount, error) { + switch len(items) { + case 0: + return false, nil, nil + case 1: + return true, &items[0], nil + default: + log.Warnf("invalid users, more than one account was associated with the same email: %v", login) + return false, nil, fmt.Errorf("multiple accounts found for login %q", login) + } +} diff --git a/modules/security/rbac/user_test.go b/modules/security/rbac/user_test.go index be4a9333f..163f138eb 100644 --- a/modules/security/rbac/user_test.go +++ b/modules/security/rbac/user_test.go @@ -23,7 +23,12 @@ package rbac -import "testing" +import ( + "strings" + "testing" + + "infini.sh/framework/core/security" +) // Weak passwords should now fail as normal validation errors instead of aborting // the request flow via panic. @@ -36,3 +41,22 @@ func TestValidateSecurePassword(t *testing.T) { t.Fatalf("expected strong password to pass validation, got %v", err) } } + +func TestResolveUserByLogin(t *testing.T) { + found, user, err := resolveUserByLogin("missing@example.org", nil) + if err != nil || found || user != nil { + t.Fatalf("expected empty result for missing user, got found=%v user=%#v err=%v", found, user, err) + } + + items := []security.UserAccount{{}} + items[0].Email = "admin@example.org" + found, user, err = resolveUserByLogin("admin@example.org", items) + if err != nil || !found || user == nil || user.Email != "admin@example.org" { + t.Fatalf("expected single user match, got found=%v user=%#v err=%v", found, user, err) + } + + _, _, err = resolveUserByLogin("dup@example.org", []security.UserAccount{{}, {}}) + if err == nil || !strings.Contains(err.Error(), "multiple accounts found") { + t.Fatalf("expected duplicate login error, got %v", err) + } +} From c4c59352a91fd781900c848533d8bd297f57b69e Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 26 May 2026 19:10:00 +0800 Subject: [PATCH 058/137] Drop legacy claim aliases Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- core/security/user_session.go | 63 ------------------------------ core/security/user_session_test.go | 46 ++++++---------------- 2 files changed, 12 insertions(+), 97 deletions(-) diff --git a/core/security/user_session.go b/core/security/user_session.go index 5451435f1..b640921d1 100644 --- a/core/security/user_session.go +++ b/core/security/user_session.go @@ -24,9 +24,7 @@ package security import ( - "encoding/json" "fmt" - "strings" "time" log "github.com/cihub/seelog" @@ -49,67 +47,6 @@ func NewUserClaims() *UserClaims { } } -type userSessionInfoAlias UserSessionInfo - -// MarshalJSON keeps the framework claims readable by older console clients while the -// token/session stack is converging onto the shared framework implementation. -func (c UserClaims) MarshalJSON() ([]byte, error) { - sessionUser := c.UserSessionInfo - if sessionUser == nil { - sessionUser = &UserSessionInfo{} - } - - claims := c.RegisteredClaims - if claims == nil { - claims = &jwt.RegisteredClaims{} - } - - return json.Marshal(struct { - *jwt.RegisteredClaims - *userSessionInfoAlias - Username string `json:"username,omitempty"` - UserID string `json:"user_id,omitempty"` - }{ - RegisteredClaims: claims, - userSessionInfoAlias: (*userSessionInfoAlias)(sessionUser), - Username: sessionUser.Login, - UserID: sessionUser.UserID, - }) -} - -// UnmarshalJSON accepts both the framework-native field names and the older console -// aliases so apps can switch validators without forcing a token-format fork first. -func (c *UserClaims) UnmarshalJSON(data []byte) error { - aux := struct { - *jwt.RegisteredClaims - *userSessionInfoAlias - Username string `json:"username,omitempty"` - UserID string `json:"user_id,omitempty"` - }{ - RegisteredClaims: &jwt.RegisteredClaims{}, - userSessionInfoAlias: &userSessionInfoAlias{}, - } - - if err := json.Unmarshal(data, &aux); err != nil { - return err - } - - sessionUser := (*UserSessionInfo)(aux.userSessionInfoAlias) - if sessionUser == nil { - sessionUser = &UserSessionInfo{} - } - if strings.TrimSpace(sessionUser.Login) == "" { - sessionUser.Login = strings.TrimSpace(aux.Username) - } - if strings.TrimSpace(sessionUser.UserID) == "" { - sessionUser.UserID = strings.TrimSpace(aux.UserID) - } - - c.RegisteredClaims = aux.RegisteredClaims - c.UserSessionInfo = sessionUser - return nil -} - // auth user info type UserSessionInfo struct { param.Parameters diff --git a/core/security/user_session_test.go b/core/security/user_session_test.go index d383b829a..e857865bb 100644 --- a/core/security/user_session_test.go +++ b/core/security/user_session_test.go @@ -25,15 +25,12 @@ package security import ( "encoding/json" - "strings" "testing" "github.com/golang-jwt/jwt/v4" ) -// Framework-issued tokens need to remain readable by console clients until the two -// stacks finish converging on a single session claim format. -func TestUserClaimsMarshalIncludesLegacyConsoleAliases(t *testing.T) { +func TestUserClaimsMarshalUsesFrameworkFields(t *testing.T) { claims := UserClaims{ RegisteredClaims: &jwt.RegisteredClaims{}, UserSessionInfo: &UserSessionInfo{ @@ -49,41 +46,22 @@ func TestUserClaimsMarshalIncludesLegacyConsoleAliases(t *testing.T) { t.Fatalf("marshal claims: %v", err) } - text := string(payload) - for _, expected := range []string{ - `"login":"admin@example.org"`, - `"username":"admin@example.org"`, - `"userid":"user-1"`, - `"user_id":"user-1"`, - } { - if !strings.Contains(text, expected) { - t.Fatalf("expected %s in %s", expected, text) - } + var data map[string]any + if err := json.Unmarshal(payload, &data); err != nil { + t.Fatalf("unmarshal claims json: %v", err) } -} -// Older console tokens only carried username/user_id, so the framework parser must -// backfill its native login/userid fields from those aliases during migration. -func TestUserClaimsUnmarshalAcceptsLegacyConsoleAliases(t *testing.T) { - var claims UserClaims - err := json.Unmarshal([]byte(`{ - "provider":"native", - "username":"admin@example.org", - "user_id":"user-1", - "roles":["admin"] - }`), &claims) - if err != nil { - t.Fatalf("unmarshal claims: %v", err) + if data["login"] != "admin@example.org" { + t.Fatalf("expected login field, got %#v", data["login"]) } - - if claims.Login != "admin@example.org" { - t.Fatalf("expected login to be backfilled from username, got %q", claims.Login) + if data["userid"] != "user-1" { + t.Fatalf("expected userid field, got %#v", data["userid"]) } - if claims.UserID != "user-1" { - t.Fatalf("expected user id to be backfilled from user_id, got %q", claims.UserID) + if _, exists := data["username"]; exists { + t.Fatalf("did not expect legacy username alias in claims: %s", payload) } - if claims.Provider != "native" { - t.Fatalf("expected provider to be preserved, got %q", claims.Provider) + if _, exists := data["user_id"]; exists { + t.Fatalf("did not expect legacy user_id alias in claims: %s", payload) } } From 856ed7d58150c64134d9dc0ac9f4f1d5b089fe31 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 26 May 2026 21:12:01 +0800 Subject: [PATCH 059/137] Protect framework roles in use Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- modules/security/rbac/role.go | 42 ++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/modules/security/rbac/role.go b/modules/security/rbac/role.go index 92c175b37..af5a7fad5 100644 --- a/modules/security/rbac/role.go +++ b/modules/security/rbac/role.go @@ -24,6 +24,7 @@ const ( errCannotUpdateOwnRole = "you can not update the roles for you" errReservedRoleName = "can not use the reserved role name" errRoleAlreadyExists = "same role name already exists" + errRoleAssignedToUsers = "role is still assigned to users" ) func GetRole(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { @@ -109,8 +110,28 @@ func DeleteRole(w http.ResponseWriter, req *http.Request, ps httprouter.Params) obj.ID = id ctx := orm.NewContextWithParent(req.Context()) ctx.DirectAccess() + + exists, err := orm.GetV2(ctx, &obj) + if err != nil { + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if !exists { + api.WriteJSON(w, api.NotFoundResponse(id), http.StatusNotFound) + return + } + inUse, err := roleHasAssignedUsers(req.Context(), obj.Name) + if err != nil { + api.WriteError(w, err.Error(), http.StatusInternalServerError) + return + } + if inUse { + api.WriteError(w, errRoleAssignedToUsers, http.StatusConflict) + return + } + ctx.Refresh = orm.WaitForRefresh - err := orm.Delete(ctx, &obj) + err = orm.Delete(ctx, &obj) if err != nil { api.WriteError(w, err.Error(), http.StatusInternalServerError) return @@ -287,3 +308,22 @@ func (provider *SecurityBackendProvider) GetPermissionKeysByRoles(ctx1 context.C } return keys } + +func roleHasAssignedUsers(ctx1 context.Context, roleName string) (bool, error) { + if roleName == "" { + return false, nil + } + + ctx := orm.NewContextWithParent(ctx1) + ctx.DirectReadAccess() + ctx.PermissionScope(security.PermissionScopePlatform) + orm.WithModel(ctx, &security.UserAccount{}) + + qb := orm.NewQuery() + qb.Must(orm.TermQuery("roles", roleName)) + err, result := elastic.SearchV2WithResultItemMapper(ctx, nil, qb, nil) + if err != nil { + return false, err + } + return result != nil && result.Total > 0, nil +} From 6ed85186ea6b777946ff07e0ec28c2e3303caf26 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 27 May 2026 09:15:48 +0800 Subject: [PATCH 060/137] improve: add check for already register --- core/kv/kv.go | 8 ++++++++ core/kv/kv_test.go | 25 +++++++++++++++++++++++++ core/orm/orm_test.go | 22 ++++++++++++++++++++++ core/orm/registry.go | 8 ++++++++ modules/elastic/module.go | 12 ++++++++++-- 5 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 core/kv/kv_test.go diff --git a/core/kv/kv.go b/core/kv/kv.go index 374e1893d..56e7662c8 100755 --- a/core/kv/kv.go +++ b/core/kv/kv.go @@ -54,6 +54,14 @@ type KVStore interface { var handler KVStore +func HasStore(name string) bool { + if stores == nil { + return false + } + _, ok := stores[name] + return ok +} + func getKVHandler() KVStore { if handler == nil { diff --git a/core/kv/kv_test.go b/core/kv/kv_test.go new file mode 100644 index 000000000..0102e97c1 --- /dev/null +++ b/core/kv/kv_test.go @@ -0,0 +1,25 @@ +package kv + +import "testing" + +func TestHasStore(t *testing.T) { + previousHandler := handler + previousStores := stores + defer func() { + handler = previousHandler + stores = previousStores + }() + + handler = nil + stores = nil + + if HasStore("elastic") { + t.Fatal("expected store lookup to be false before registration") + } + + Register("elastic", nil) + + if !HasStore("elastic") { + t.Fatal("expected store lookup to be true after registration") + } +} diff --git a/core/orm/orm_test.go b/core/orm/orm_test.go index 82491758f..d682aa128 100644 --- a/core/orm/orm_test.go +++ b/core/orm/orm_test.go @@ -117,6 +117,28 @@ func TestSetFieldTimeValue(t *testing.T) { } +func TestHasAdapter(t *testing.T) { + previousHandler := handler + previousAdapters := adapters + defer func() { + handler = previousHandler + adapters = previousAdapters + }() + + handler = nil + adapters = nil + + if HasAdapter("elastic") { + t.Fatal("expected adapter lookup to be false before registration") + } + + Register("elastic", nil) + + if !HasAdapter("elastic") { + t.Fatal("expected adapter lookup to be true after registration") + } +} + //func TestSetFieldTimeValue1(t *testing.T) { // t1:=time.Now() // a:=struct { diff --git a/core/orm/registry.go b/core/orm/registry.go index ab3b4513b..bf98277d2 100644 --- a/core/orm/registry.go +++ b/core/orm/registry.go @@ -55,6 +55,14 @@ func HasHandler() bool { return handler != nil } +func HasAdapter(name string) bool { + if adapters == nil { + return false + } + _, ok := adapters[name] + return ok +} + func getHandler() ORM { if handler == nil { panic(errors.New("ORM handler is not registered")) diff --git a/modules/elastic/module.go b/modules/elastic/module.go index 56ad1ca42..cd7b8e733 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -415,7 +415,11 @@ func (module *ElasticModule) Start() error { } else { client := elastic.GetClient(systemID) handler := ElasticORM{Client: client, Config: moduleConfig.ORMConfig} - orm.Register("elastic", &handler) + if orm.HasAdapter("elastic") { + log.Debug("skip duplicate elastic ORM registration") + } else { + orm.Register("elastic", &handler) + } } } @@ -425,7 +429,11 @@ func (module *ElasticModule) Start() error { } else { client := elastic.GetClient(systemID) module.storeHandler = &ElasticStore{Client: client, Config: moduleConfig.StoreConfig} - kv.Register("elastic", module.storeHandler) + if kv.HasStore("elastic") { + log.Debug("skip duplicate elastic store registration") + } else { + kv.Register("elastic", module.storeHandler) + } } } From 202bf91f142a2414373afa25ea29b1b6ab6695e9 Mon Sep 17 00:00:00 2001 From: Medcl Date: Fri, 22 May 2026 19:57:56 +0800 Subject: [PATCH 061/137] refactor: refactoring to simplify go modules (#300) * refactor: maintain changed vendors * refactor: add logging api * refactor: refactoring imports * chore: update docs * chore: make seelog easier * chore: remove vendor in makefile * fix(test): stabilize simplify_go_modules unit-test failures in seelog/statsd/tencentcloud (#302) * Initial plan * chore: outline plan for unit test fixes Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/ba6d570b-571b-42db-94e8-4791d77c6ec1 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * fix(test): stabilize unit tests on simplify_go_modules Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/ba6d570b-571b-42db-94e8-4791d77c6ec1 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * fix(test): harden async flush and socket listener tests Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/ba6d570b-571b-42db-94e8-4791d77c6ec1 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * chore: upgrade jwt to v4 * chore: update ci pipelines * chore: remove unused bench test * chore: fix tests * test(rotate): remove invalid example test --------- Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> --- .github/workflows/integration-test.yml | 9 +- .github/workflows/unit_test.yml | 9 +- Makefile | 47 +- README.md | 26 +- app.go | 13 +- cmd/plugin-discovery/main.go | 2 +- core/api/certs.go | 11 +- core/log/log.go | 34 + core/logging/log_test.go | 45 - core/pipeline/pools_benchmark_test.go | 131 - core/pipeline/pools_test.go | 384 -- core/rotate/rotate_test.go | 22 - core/rotate/writer_test.go | 375 -- core/task/chrono/task_test.go | 15 - dev.go | 7 +- .../development/create_new_application.md | 11 +- .../development/setup_golang_environment.md | 26 +- docs/content.en/docs/references/makefile.md | 30 +- go.mod | 54 +- go.sum | 37 +- lib/bytebufferpool/bytebuffer_test.go | 13 - lib/bytebufferpool/pool_test.go | 18 - lib/cache/cache_test.go | 14 +- lib/cache/layeredcache_test.go | 14 +- lib/fasthttp/allocation_test.go | 94 - lib/fasthttp/args_test.go | 623 --- lib/fasthttp/args_timing_test.go | 30 - lib/fasthttp/brotli_test.go | 102 - lib/fasthttp/bytesconv_32_test.go | 60 - lib/fasthttp/bytesconv_64_test.go | 62 - lib/fasthttp/bytesconv_test.go | 349 -- lib/fasthttp/bytesconv_timing_test.go | 165 - lib/fasthttp/client_example_test.go | 39 - lib/fasthttp/client_test.go | 2912 ------------ lib/fasthttp/client_timing_test.go | 642 --- lib/fasthttp/client_timing_wait_test.go | 167 - lib/fasthttp/client_unix_test.go | 136 - lib/fasthttp/coarseTime_test.go | 37 - lib/fasthttp/compress_test.go | 232 - lib/fasthttp/cookie_test.go | 414 -- lib/fasthttp/cookie_timing_test.go | 35 - lib/fasthttp/expvarhandler/expvar_test.go | 69 - lib/fasthttp/fasthttpadaptor/adaptor_test.go | 172 - .../fasthttputil/inmemory_listener_test.go | 192 - .../inmemory_listener_timing_test.go | 108 - lib/fasthttp/fasthttputil/pipeconns_test.go | 360 -- lib/fasthttp/fs_example_test.go | 28 - lib/fasthttp/fs_handler_example_test.go | 47 - lib/fasthttp/fs_test.go | 853 ---- lib/fasthttp/header_regression_test.go | 91 - lib/fasthttp/header_test.go | 2885 ------------ lib/fasthttp/header_timing_test.go | 221 - lib/fasthttp/http_test.go | 2966 ------------ lib/fasthttp/lbclient_example_test.go | 42 - lib/fasthttp/peripconn_test.go | 63 - lib/fasthttp/prefork/prefork_test.go | 228 - lib/fasthttp/request_context_test.go | 13 - ...estctx_setbodystreamwriter_example_test.go | 32 - .../reuseport/reuseport_example_test.go | 24 - lib/fasthttp/reuseport/reuseport_test.go | 49 - lib/fasthttp/server_example_test.go | 156 - lib/fasthttp/server_test.go | 4182 ----------------- lib/fasthttp/server_timing_test.go | 461 -- lib/fasthttp/stackless/func_test.go | 90 - lib/fasthttp/stackless/func_timing_test.go | 40 - lib/fasthttp/stackless/writer_test.go | 130 - lib/fasthttp/status_test.go | 24 - lib/fasthttp/status_timing_test.go | 29 - lib/fasthttp/stream_test.go | 106 - lib/fasthttp/stream_timing_test.go | 70 - lib/fasthttp/streaming_test.go | 262 -- lib/fasthttp/uri_test.go | 490 -- lib/fasthttp/uri_timing_test.go | 49 - lib/fasthttp/uri_windows_test.go | 15 - lib/fasthttp/userdata_test.go | 121 - lib/fasthttp/userdata_timing_test.go | 48 - lib/fasthttp/workerpool_test.go | 177 - lib/fastjson_marshal/marshaler_test.go | 68 - lib/gomail/LICENSE | 20 + lib/gomail/README.md | 92 + lib/gomail/auth.go | 49 + lib/gomail/auth_test.go | 100 + lib/gomail/doc.go | 5 + lib/gomail/example_test.go | 223 + lib/gomail/message.go | 322 ++ lib/gomail/message_test.go | 745 +++ lib/gomail/mime.go | 22 + lib/gomail/mime_go14.go | 26 + lib/gomail/send.go | 116 + lib/gomail/send_test.go | 80 + lib/gomail/smtp.go | 209 + lib/gomail/smtp_test.go | 292 ++ lib/gomail/writeto.go | 306 ++ lib/guardian/auth/strategies/ldap/ldap.go | 5 +- lib/guardian/go.sum | 1 - lib/lock_free/queue/esQueue_test.go | 451 -- lib/router/router_test.go | 311 -- lib/seelog/LICENSE.txt | 24 + lib/seelog/README.markdown | 116 + lib/seelog/behavior_adaptive_test.go | 124 + lib/seelog/behavior_adaptivelogger.go | 130 + lib/seelog/behavior_asynclogger.go | 142 + lib/seelog/behavior_asyncloop_test.go | 133 + lib/seelog/behavior_asynclooplogger.go | 69 + lib/seelog/behavior_asynctimer_test.go | 83 + lib/seelog/behavior_asynctimerlogger.go | 82 + lib/seelog/behavior_synclogger.go | 75 + lib/seelog/behavior_synclogger_test.go | 81 + lib/seelog/cfg_config.go | 188 + lib/seelog/cfg_errors.go | 61 + lib/seelog/cfg_logconfig.go | 141 + lib/seelog/cfg_logconfig_test.go | 99 + lib/seelog/cfg_parser.go | 1238 +++++ lib/seelog/cfg_parser_test.go | 1096 +++++ lib/seelog/common_closer.go | 25 + lib/seelog/common_constraints.go | 162 + lib/seelog/common_constraints_test.go | 196 + lib/seelog/common_context.go | 194 + lib/seelog/common_context_test.go | 127 + lib/seelog/common_exception.go | 194 + lib/seelog/common_exception_test.go | 98 + lib/seelog/common_flusher.go | 31 + lib/seelog/common_loglevel.go | 81 + lib/seelog/dispatch_custom.go | 243 + lib/seelog/dispatch_customdispatcher_test.go | 177 + lib/seelog/dispatch_dispatcher.go | 189 + lib/seelog/dispatch_filterdispatcher.go | 66 + lib/seelog/dispatch_filterdispatcher_test.go | 67 + lib/seelog/dispatch_splitdispatcher.go | 47 + lib/seelog/dispatch_splitdispatcher_test.go | 64 + lib/seelog/doc.go | 190 + lib/seelog/format.go | 461 ++ lib/seelog/format_test.go | 236 + lib/seelog/go.mod | 3 + lib/seelog/internals_baseerror.go | 10 + lib/seelog/internals_byteverifiers_test.go | 118 + lib/seelog/internals_fsutils.go | 403 ++ lib/seelog/internals_xmlnode.go | 175 + lib/seelog/internals_xmlnode_test.go | 196 + lib/seelog/log.go | 313 ++ lib/seelog/logger.go | 370 ++ lib/seelog/writers_bufferedwriter.go | 178 + lib/seelog/writers_bufferedwriter_test.go | 94 + lib/seelog/writers_connwriter.go | 144 + lib/seelog/writers_consolewriter.go | 47 + lib/seelog/writers_filewriter.go | 92 + lib/seelog/writers_filewriter_test.go | 254 + lib/seelog/writers_formattedwriter.go | 62 + lib/seelog/writers_formattedwriter_test.go | 65 + lib/seelog/writers_rollingfilewriter.go | 625 +++ lib/seelog/writers_rollingfilewriter_test.go | 99 + lib/seelog/writers_smtpwriter.go | 214 + lib/statsd/LICENSE | 21 + lib/statsd/README.md | 77 + lib/statsd/bufferedclient.go | 200 + lib/statsd/bufferedclient_test.go | 86 + lib/statsd/client.go | 257 + lib/statsd/client_test.go | 315 ++ lib/statsd/event/absolute.go | 58 + lib/statsd/event/fabsolute.go | 58 + lib/statsd/event/fgauge.go | 64 + lib/statsd/event/fgaugedelta.go | 64 + lib/statsd/event/gauge.go | 64 + lib/statsd/event/gaugedelta.go | 64 + lib/statsd/event/increment.go | 53 + lib/statsd/event/interface.go | 27 + lib/statsd/event/precisiontiming.go | 78 + lib/statsd/event/precisiontiming_test.go | 21 + lib/statsd/event/timing.go | 88 + lib/statsd/event/timing_test.go | 20 + lib/statsd/event/total.go | 53 + lib/statsd/interface.go | 22 + lib/statsd/noopclient.go | 80 + lib/tencentcloud/LICENSE | 22 + lib/tencentcloud/README.md | 23 + lib/tencentcloud/client.go | 201 + lib/tencentcloud/provider.go | 63 + lib/tencentcloud/provider_test.go | 61 + lib/tencentcloud/signer.go | 66 + lib/tencentcloud/types.go | 98 + modules/elastic/adapter/easysearch/v1.go | 36 +- modules/elastic/adapter/elasticsearch/V6.6.go | 6 +- modules/elastic/adapter/elasticsearch/v0.go | 18 +- modules/elastic/adapter/elasticsearch/v7.go | 6 +- modules/elastic/adapter/elasticsearch/v8.go | 2 +- modules/elastic/adapter/opensearch/v1.go | 6 +- modules/stats/simple_test.go | 71 - plugins/badger/module_test.go | 37 +- plugins/smtp/smtp.go | 2 +- plugins/stats_statsd/statsd.go | 5 +- 190 files changed, 15330 insertions(+), 23137 deletions(-) create mode 100644 core/log/log.go mode change 100755 => 100644 core/logging/log_test.go delete mode 100755 core/pipeline/pools_benchmark_test.go delete mode 100755 core/pipeline/pools_test.go delete mode 100644 lib/fasthttp/allocation_test.go delete mode 100644 lib/fasthttp/args_test.go delete mode 100644 lib/fasthttp/args_timing_test.go delete mode 100644 lib/fasthttp/brotli_test.go delete mode 100644 lib/fasthttp/bytesconv_32_test.go delete mode 100644 lib/fasthttp/bytesconv_64_test.go delete mode 100644 lib/fasthttp/bytesconv_test.go delete mode 100644 lib/fasthttp/bytesconv_timing_test.go delete mode 100644 lib/fasthttp/client_example_test.go delete mode 100644 lib/fasthttp/client_test.go delete mode 100644 lib/fasthttp/client_timing_test.go delete mode 100644 lib/fasthttp/client_timing_wait_test.go delete mode 100644 lib/fasthttp/client_unix_test.go delete mode 100644 lib/fasthttp/coarseTime_test.go delete mode 100644 lib/fasthttp/compress_test.go delete mode 100644 lib/fasthttp/cookie_test.go delete mode 100644 lib/fasthttp/cookie_timing_test.go delete mode 100644 lib/fasthttp/expvarhandler/expvar_test.go delete mode 100644 lib/fasthttp/fasthttpadaptor/adaptor_test.go delete mode 100644 lib/fasthttp/fasthttputil/inmemory_listener_test.go delete mode 100644 lib/fasthttp/fasthttputil/inmemory_listener_timing_test.go delete mode 100644 lib/fasthttp/fasthttputil/pipeconns_test.go delete mode 100644 lib/fasthttp/fs_example_test.go delete mode 100644 lib/fasthttp/fs_handler_example_test.go delete mode 100644 lib/fasthttp/fs_test.go delete mode 100644 lib/fasthttp/header_regression_test.go delete mode 100644 lib/fasthttp/header_test.go delete mode 100644 lib/fasthttp/header_timing_test.go delete mode 100644 lib/fasthttp/http_test.go delete mode 100644 lib/fasthttp/lbclient_example_test.go delete mode 100644 lib/fasthttp/peripconn_test.go delete mode 100644 lib/fasthttp/prefork/prefork_test.go delete mode 100644 lib/fasthttp/request_context_test.go delete mode 100644 lib/fasthttp/requestctx_setbodystreamwriter_example_test.go delete mode 100644 lib/fasthttp/reuseport/reuseport_example_test.go delete mode 100644 lib/fasthttp/reuseport/reuseport_test.go delete mode 100644 lib/fasthttp/server_example_test.go delete mode 100644 lib/fasthttp/server_test.go delete mode 100644 lib/fasthttp/server_timing_test.go delete mode 100644 lib/fasthttp/stackless/func_test.go delete mode 100644 lib/fasthttp/stackless/func_timing_test.go delete mode 100644 lib/fasthttp/stackless/writer_test.go delete mode 100644 lib/fasthttp/status_test.go delete mode 100644 lib/fasthttp/status_timing_test.go delete mode 100644 lib/fasthttp/stream_test.go delete mode 100644 lib/fasthttp/stream_timing_test.go delete mode 100644 lib/fasthttp/streaming_test.go delete mode 100644 lib/fasthttp/uri_test.go delete mode 100644 lib/fasthttp/uri_timing_test.go delete mode 100644 lib/fasthttp/uri_windows_test.go delete mode 100644 lib/fasthttp/userdata_test.go delete mode 100644 lib/fasthttp/userdata_timing_test.go delete mode 100644 lib/fasthttp/workerpool_test.go create mode 100644 lib/gomail/LICENSE create mode 100644 lib/gomail/README.md create mode 100644 lib/gomail/auth.go create mode 100644 lib/gomail/auth_test.go create mode 100644 lib/gomail/doc.go create mode 100644 lib/gomail/example_test.go create mode 100644 lib/gomail/message.go create mode 100644 lib/gomail/message_test.go create mode 100644 lib/gomail/mime.go create mode 100644 lib/gomail/mime_go14.go create mode 100644 lib/gomail/send.go create mode 100644 lib/gomail/send_test.go create mode 100644 lib/gomail/smtp.go create mode 100644 lib/gomail/smtp_test.go create mode 100644 lib/gomail/writeto.go delete mode 100755 lib/lock_free/queue/esQueue_test.go create mode 100644 lib/seelog/LICENSE.txt create mode 100644 lib/seelog/README.markdown create mode 100644 lib/seelog/behavior_adaptive_test.go create mode 100644 lib/seelog/behavior_adaptivelogger.go create mode 100644 lib/seelog/behavior_asynclogger.go create mode 100644 lib/seelog/behavior_asyncloop_test.go create mode 100644 lib/seelog/behavior_asynclooplogger.go create mode 100644 lib/seelog/behavior_asynctimer_test.go create mode 100644 lib/seelog/behavior_asynctimerlogger.go create mode 100644 lib/seelog/behavior_synclogger.go create mode 100644 lib/seelog/behavior_synclogger_test.go create mode 100644 lib/seelog/cfg_config.go create mode 100644 lib/seelog/cfg_errors.go create mode 100644 lib/seelog/cfg_logconfig.go create mode 100644 lib/seelog/cfg_logconfig_test.go create mode 100644 lib/seelog/cfg_parser.go create mode 100644 lib/seelog/cfg_parser_test.go create mode 100644 lib/seelog/common_closer.go create mode 100644 lib/seelog/common_constraints.go create mode 100644 lib/seelog/common_constraints_test.go create mode 100644 lib/seelog/common_context.go create mode 100644 lib/seelog/common_context_test.go create mode 100644 lib/seelog/common_exception.go create mode 100644 lib/seelog/common_exception_test.go create mode 100644 lib/seelog/common_flusher.go create mode 100644 lib/seelog/common_loglevel.go create mode 100644 lib/seelog/dispatch_custom.go create mode 100644 lib/seelog/dispatch_customdispatcher_test.go create mode 100644 lib/seelog/dispatch_dispatcher.go create mode 100644 lib/seelog/dispatch_filterdispatcher.go create mode 100644 lib/seelog/dispatch_filterdispatcher_test.go create mode 100644 lib/seelog/dispatch_splitdispatcher.go create mode 100644 lib/seelog/dispatch_splitdispatcher_test.go create mode 100644 lib/seelog/doc.go create mode 100644 lib/seelog/format.go create mode 100644 lib/seelog/format_test.go create mode 100644 lib/seelog/go.mod create mode 100644 lib/seelog/internals_baseerror.go create mode 100644 lib/seelog/internals_byteverifiers_test.go create mode 100644 lib/seelog/internals_fsutils.go create mode 100644 lib/seelog/internals_xmlnode.go create mode 100644 lib/seelog/internals_xmlnode_test.go create mode 100644 lib/seelog/log.go create mode 100644 lib/seelog/logger.go create mode 100644 lib/seelog/writers_bufferedwriter.go create mode 100644 lib/seelog/writers_bufferedwriter_test.go create mode 100644 lib/seelog/writers_connwriter.go create mode 100644 lib/seelog/writers_consolewriter.go create mode 100644 lib/seelog/writers_filewriter.go create mode 100644 lib/seelog/writers_filewriter_test.go create mode 100644 lib/seelog/writers_formattedwriter.go create mode 100644 lib/seelog/writers_formattedwriter_test.go create mode 100644 lib/seelog/writers_rollingfilewriter.go create mode 100644 lib/seelog/writers_rollingfilewriter_test.go create mode 100644 lib/seelog/writers_smtpwriter.go create mode 100644 lib/statsd/LICENSE create mode 100644 lib/statsd/README.md create mode 100644 lib/statsd/bufferedclient.go create mode 100644 lib/statsd/bufferedclient_test.go create mode 100644 lib/statsd/client.go create mode 100644 lib/statsd/client_test.go create mode 100644 lib/statsd/event/absolute.go create mode 100644 lib/statsd/event/fabsolute.go create mode 100644 lib/statsd/event/fgauge.go create mode 100644 lib/statsd/event/fgaugedelta.go create mode 100644 lib/statsd/event/gauge.go create mode 100644 lib/statsd/event/gaugedelta.go create mode 100644 lib/statsd/event/increment.go create mode 100644 lib/statsd/event/interface.go create mode 100644 lib/statsd/event/precisiontiming.go create mode 100644 lib/statsd/event/precisiontiming_test.go create mode 100644 lib/statsd/event/timing.go create mode 100644 lib/statsd/event/timing_test.go create mode 100644 lib/statsd/event/total.go create mode 100644 lib/statsd/interface.go create mode 100644 lib/statsd/noopclient.go create mode 100644 lib/tencentcloud/LICENSE create mode 100644 lib/tencentcloud/README.md create mode 100644 lib/tencentcloud/client.go create mode 100644 lib/tencentcloud/provider.go create mode 100644 lib/tencentcloud/provider_test.go create mode 100644 lib/tencentcloud/signer.go create mode 100644 lib/tencentcloud/types.go delete mode 100644 modules/stats/simple_test.go diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 52a669a68..15f8bc215 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -66,13 +66,6 @@ jobs: repository: infinilabs/loadgen path: loadgen - - name: Checkout framework-vendor - uses: actions/checkout@v4 - with: - ref: main - repository: infinilabs/framework-vendor - path: vendor - - name: Set up nodejs toolchain uses: actions/setup-node@v4 with: @@ -141,7 +134,7 @@ jobs: # for products for p in console gateway agent loadgen; do cd $WORK/$p && echo Compiling $p at $PWD ... - OFFLINE_BUILD=true GOMODULE=false make tidy build + OFFLINE_BUILD=true make tidy build done - name: Prepare console config diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index afe06b8d6..366ba36df 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -20,13 +20,6 @@ jobs: with: path: framework - - name: Checkout framework-vendor - uses: actions/checkout@v4 - with: - ref: main - repository: infinilabs/framework-vendor - path: vendor - - name: Set up go toolchain uses: actions/setup-go@v5 with: @@ -56,4 +49,4 @@ jobs: # for unit test cd $WORK echo Testing code at $PWD ... - OFFLINE_BUILD=true GOMODULE=false CI=true make tidy test \ No newline at end of file + OFFLINE_BUILD=true CI=true make tidy test \ No newline at end of file diff --git a/Makefile b/Makefile index 2a1243ab9..daeada38f 100755 --- a/Makefile +++ b/Makefile @@ -54,38 +54,21 @@ ifeq "$(FRAMEWORK_BRANCH)" "" FRAMEWORK_BRANCH := main endif -FRAMEWORK_VENDOR_FOLDER ?= $(FRAMEWORK_FOLDER)/../vendor/ -FRAMEWORK_VENDOR_REPO ?= https://github.com/infinilabs/framework-vendor.git -ifeq "$(FRAMEWORK_VENDOR_BRANCH)" "" - FRAMEWORK_VENDOR_BRANCH := main -endif - ifneq "$(DEV)" "" FRAMEWORK_DEVEL_BUILD := -tags dev endif -# Adjust the vendor priority -PREFER_MANAGED_VENDOR ?= true -NEWGOPATH:= $(FRAMEWORK_VENDOR_FOLDER):$(GOPATH) -ifneq "$(PREFER_MANAGED_VENDOR)" "true" - NEWGOPATH:= $(GOPATH):$(FRAMEWORK_VENDOR_FOLDER) -endif - GO := go -GOMODULE ?= true -ifneq "$(GOMODULE)" "true" - GO := GO15VENDOREXPERIMENT="1" GO111MODULE=off go -endif -GOBUILD := GOPATH=$(NEWGOPATH) CGO_ENABLED=$(APP_NEED_CGO) GRPC_GO_REQUIRE_HANDSHAKE=off $(GO) build -a $(FRAMEWORK_DEVEL_BUILD) -gcflags=all="-l -B" -ldflags '-static' -ldflags='-s -w' -gcflags "-m" --work $(GOBUILD_FLAGS) -GOBUILDDBG := GOPATH=$(NEWGOPATH) CGO_ENABLED=$(APP_NEED_CGO) GRPC_GO_REQUIRE_HANDSHAKE=off $(GO) build -a $(FRAMEWORK_DEVEL_BUILD) -ldflags -v -gcflags "all=-N -l" --work $(GOBUILD_FLAGS) -GOBUILDNCGO := GOPATH=$(NEWGOPATH) CGO_ENABLED=1 $(GO) build -ldflags -s $(GOBUILD_FLAGS) -GOTEST := GOPATH=$(NEWGOPATH) CGO_ENABLED=$(APP_NEED_CGO) $(GO) test -ldflags -s +GOBUILD := CGO_ENABLED=$(APP_NEED_CGO) $(GO) build -a $(FRAMEWORK_DEVEL_BUILD) -gcflags=all="-l -B" -ldflags '-static' -ldflags='-s -w' -gcflags "-m" --work $(GOBUILD_FLAGS) +GOBUILDDBG := CGO_ENABLED=$(APP_NEED_CGO) $(GO) build -a $(FRAMEWORK_DEVEL_BUILD) -ldflags -v -gcflags "all=-N -l" --work $(GOBUILD_FLAGS) +GOBUILDNCGO := CGO_ENABLED=1 $(GO) build -ldflags -s $(GOBUILD_FLAGS) +GOTEST := CGO_ENABLED=$(APP_NEED_CGO) $(GO) test -ldflags -s ARCH := "`uname -s`" LINUX := "Linux" MAC := "Darwin" -GO_FILES=$(find . -iname '*.go' | grep -v /vendor/) -PKGS=$(go list ./... | grep -v /vendor/) +GO_FILES=$(find . -iname '*.go') +PKGS=$(go list ./...) FRAMEWORK_OFFLINE_BUILD := "" ifneq "$(OFFLINE_BUILD)" "" @@ -100,12 +83,9 @@ default: build-race env: @echo OLDGOPATH：$(OLDGOPATH) @echo GOPATH：$(GOPATH) - @echo NEWGOPATH：$(NEWGOPATH) @echo INFINI_BASE_FOLDER：$(INFINI_BASE_FOLDER) @echo FRAMEWORK_FOLDER：$(FRAMEWORK_FOLDER) @echo FRAMEWORK_REPO：$(FRAMEWORK_REPO) - @echo FRAMEWORK_VENDOR_FOLDER：$(FRAMEWORK_VENDOR_FOLDER) - @echo FRAMEWORK_VENDOR_REPO：$(FRAMEWORK_VENDOR_REPO) build: config $(GOBUILD) -o $(OUTPUT_DIR)/$(APP_NAME) @@ -262,10 +242,7 @@ cross-build-all-platform: clean config build-bsd build-linux build-darwin build- format: @echo "formatting code" - find . -type f -name '*.go' \ - -not -path './vendor/*' \ - -not -path './.git/*' \ - -print0 | xargs -0 gofmt -w + $(GO) fmt $$($(GO) list ./...) test: config $(GOTEST) -v $(GOFLAGS) -timeout 30m ./... @@ -300,14 +277,10 @@ init: @mkdir -p $(INFINI_BASE_FOLDER) @echo "framework path: " $(FRAMEWORK_FOLDER) @if [ ! -d $(FRAMEWORK_FOLDER) ]; then echo "framework does not exist";(cd $(INFINI_BASE_FOLDER) && git clone -b $(FRAMEWORK_BRANCH) $(FRAMEWORK_REPO) framework ) fi - @if [ ! -d $(FRAMEWORK_VENDOR_FOLDER) ]; then echo "framework vendor does not exist";(cd $(INFINI_BASE_FOLDER) && git clone -b $(FRAMEWORK_VENDOR_BRANCH) $(FRAMEWORK_VENDOR_REPO) $(FRAMEWORK_VENDOR_FOLDER)) fi @if [ "" == $(FRAMEWORK_OFFLINE_BUILD) ]; then (cd $(FRAMEWORK_FOLDER) && git checkout $(FRAMEWORK_BRANCH) && git pull origin $(FRAMEWORK_BRANCH)); fi; - @if [ "" == $(FRAMEWORK_OFFLINE_BUILD) ]; then (cd $(FRAMEWORK_VENDOR_FOLDER) && git checkout $(FRAMEWORK_VENDOR_BRANCH) && git pull origin $(FRAMEWORK_VENDOR_BRANCH)); fi; @# Extract the latest commit hash from the framework repository @(cd $(FRAMEWORK_FOLDER) && git rev-parse HEAD > $(FRAMEWORK_FOLDER)/.latest_commit_hash.txt) - @(cd $(FRAMEWORK_VENDOR_FOLDER) && git rev-parse HEAD > $(FRAMEWORK_VENDOR_FOLDER)/.latest_commit_hash.txt) @echo "Framework commit hash updated: " && cat $(FRAMEWORK_FOLDER)/.latest_commit_hash.txt - @echo "Framework vendor commit hash updated: " && cat $(FRAMEWORK_VENDOR_FOLDER)/.latest_commit_hash.txt update-generated-framework-info: @echo "generating framework info" @@ -315,8 +288,7 @@ update-generated-framework-info: @# Generate the framework info file @(cd $(FRAMEWORK_FOLDER) && \ LATEST_COMMIT_LOG=$$(cat .latest_commit_hash.txt) && \ - VENDOR_COMMIT_LOG=$$(cat $(FRAMEWORK_VENDOR_FOLDER)/.latest_commit_hash.txt) && \ - echo -e "package config\n\nconst LastFrameworkCommitLog = \"$$LATEST_COMMIT_LOG\"\nconst LastFrameworkVendorCommitLog = \"$$VENDOR_COMMIT_LOG\"" > config/generated_framework-info.go) + echo -e "package config\n\nconst LastFrameworkCommitLog = \"$$LATEST_COMMIT_LOG\"\nconst LastFrameworkVendorCommitLog = \"N/A\"" > config/generated_framework-info.go) update-generated-file: update-generated-framework-info @echo "generating application info" @@ -329,8 +301,7 @@ update-generated-file: update-generated-framework-info restore-generated-framework-info: @echo "restore framework info" - @( cd $(FRAMEWORK_FOLDER) && echo -e "package config\n\nconst LastFrameworkCommitLog = \"N/A\"" > config/generated_framework-info.go) - @( cd $(FRAMEWORK_FOLDER) && echo -e "\nconst LastFrameworkVendorCommitLog = \"N/A\"" >> config/generated_framework-info.go ) + @( cd $(FRAMEWORK_FOLDER) && echo -e "package config\n\nconst LastFrameworkCommitLog = \"N/A\"\nconst LastFrameworkVendorCommitLog = \"N/A\"" > config/generated_framework-info.go) restore-generated-file: restore-generated-framework-info @echo "restore application info" diff --git a/README.md b/README.md index f61dba721..857d7142b 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,28 @@ # INFINI Framework ## Requirements -- Golang v1.11+ \ No newline at end of file +- Go 1.21+ + +## Dependencies + +All dependencies are managed via Go modules (`go.mod`). There is no external vendor repository required. + +### Internalized Libraries + +The following forked/patched libraries live under `lib/` as part of this module: + +| Directory | Origin | Notes | +|-----------|--------|-------| +| `lib/seelog` | `github.com/cihub/seelog` | Logging backend; exposed via `replace` directive so existing imports keep working | +| `lib/statsd` | `github.com/quipo/statsd` | StatsD client with custom buffering changes | +| `lib/gomail` | `gopkg.in/gomail.v2` | SMTP client with `NewDialerWithTimeout` extension | +| `lib/tencentcloud` | `github.com/libdns/tencentcloud` | DNS provider with custom signer and types | + +### Building + +```bash +make build # production build +make build-dev # development build (includes -tags dev) +``` + +No `GOPATH` manipulation or vendor repository checkout is needed. Standard `go build` works directly. \ No newline at end of file diff --git a/app.go b/app.go index af1ef24e5..7b3f74e8b 100755 --- a/app.go +++ b/app.go @@ -31,11 +31,6 @@ import ( "context" "flag" "fmt" - "github.com/fsnotify/fsnotify" - "github.com/shirou/gopsutil/v3/process" - "infini.sh/framework/core/task" - "infini.sh/framework/core/wrapper/taskset" - "infini.sh/framework/modules/configs/client" "os" "os/signal" "path/filepath" @@ -45,7 +40,12 @@ import ( "syscall" "time" - log "github.com/cihub/seelog" + "github.com/fsnotify/fsnotify" + "github.com/shirou/gopsutil/v3/process" + "infini.sh/framework/core/task" + "infini.sh/framework/core/wrapper/taskset" + "infini.sh/framework/modules/configs/client" + "github.com/kardianos/service" "infini.sh/framework/core/config" "infini.sh/framework/core/daemon" @@ -53,6 +53,7 @@ import ( "infini.sh/framework/core/errors" "infini.sh/framework/core/global" "infini.sh/framework/core/keystore" + "infini.sh/framework/core/log" _ "infini.sh/framework/core/logging" "infini.sh/framework/core/logging/logger" "infini.sh/framework/core/module" diff --git a/cmd/plugin-discovery/main.go b/cmd/plugin-discovery/main.go index 9245c225c..ae0a40060 100644 --- a/cmd/plugin-discovery/main.go +++ b/cmd/plugin-discovery/main.go @@ -126,7 +126,7 @@ func main() { } func usageFlag() { - fmt.Fprintf(os.Stderr, usageText) + fmt.Fprintf(os.Stderr, "%s", usageText) flag.PrintDefaults() } diff --git a/core/api/certs.go b/core/api/certs.go index d3edf6eeb..cd4bcb0d9 100644 --- a/core/api/certs.go +++ b/core/api/certs.go @@ -32,19 +32,20 @@ import ( "crypto/tls" "crypto/x509" "encoding/pem" + "io/ioutil" + "os" + "path" + "time" + "github.com/caddyserver/certmagic" "github.com/cihub/seelog" log "github.com/cihub/seelog" - "github.com/libdns/tencentcloud" "go.uber.org/zap" "infini.sh/framework/core/config" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" "infini.sh/framework/core/util" - "io/ioutil" - "os" - "path" - "time" + "infini.sh/framework/lib/tencentcloud" ) func GetServerTLSConfig(tlsCfg *config.TLSConfig) (*tls.Config, error) { diff --git a/core/log/log.go b/core/log/log.go new file mode 100644 index 000000000..d257c9893 --- /dev/null +++ b/core/log/log.go @@ -0,0 +1,34 @@ +package log + +// Package log provides the main logging entrypoint for the framework. +// It wraps the internal seelog implementation. + +import ( + "github.com/cihub/seelog" +) + +type LoggerInterface = seelog.LoggerInterface + +// Logger is the main logger instance. +var Logger = seelog.Default + +// Convenience wrappers for seelog functions +var ( + UseLogger = seelog.UseLogger + ReplaceLogger = seelog.ReplaceLogger + Flush = seelog.Flush + + Trace = seelog.Trace + Debug = seelog.Debug + Info = seelog.Info + Warn = seelog.Warn + Error = seelog.Error + Critical = seelog.Critical + + Tracef = seelog.Tracef + Debugf = seelog.Debugf + Infof = seelog.Infof + Warnf = seelog.Warnf + Errorf = seelog.Errorf + Criticalf = seelog.Criticalf +) diff --git a/core/logging/log_test.go b/core/logging/log_test.go old mode 100755 new mode 100644 index 46457eb98..498b69b4f --- a/core/logging/log_test.go +++ b/core/logging/log_test.go @@ -38,48 +38,3 @@ limitations under the License. */ package logging - -import ( - "log" - "os" - "runtime" - "testing" -) - -func TestLogging(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - testMain() -} - -func testMain() { - DumpAllThread() - Logging1() -} - -func Logging1() { - pc, file, line, ok := runtime.Caller(2) - log.Println(pc) - log.Println(file) - log.Println(line) - log.Println(ok) - f := runtime.FuncForPC(pc) - log.Println(f.Name()) - - pc, file, line, ok = runtime.Caller(0) - log.Println(pc) - log.Println(file) - log.Println(line) - log.Println(ok) - f = runtime.FuncForPC(pc) - log.Println(f.Name()) - - pc, file, line, ok = runtime.Caller(1) - log.Println(pc) - log.Println(file) - log.Println(line) - log.Println(ok) - f = runtime.FuncForPC(pc) - log.Println(f.Name()) -} diff --git a/core/pipeline/pools_benchmark_test.go b/core/pipeline/pools_benchmark_test.go deleted file mode 100755 index afde65233..000000000 --- a/core/pipeline/pools_benchmark_test.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (C) INFINI Labs & INFINI LIMITED. -// -// The INFINI Framework is offered under the GNU Affero General Public License v3.0 -// and as commercial software. -// -// For commercial licensing, contact us at: -// - Website: infinilabs.com -// - Email: hello@infini.ltd -// -// Open Source licensed under AGPL V3: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -// MIT License - -// Copyright (c) 2018 Andy Pan - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package pipeline - -import ( - "runtime" - "sync" - "testing" - "time" -) - -const ( - RunTimes = 1000000 - BenchParam = 10 - BenchAntsSize = 200000 - DefaultExpiredTime = 10 * time.Second -) - -var demoFunc = &Task{Handler: demo} - -func demo(ctx *Context, v ...interface{}) { - time.Sleep(time.Duration(BenchParam) * time.Millisecond) -} - -func demoPoolFunc(args interface{}) { - n := args.(int) - time.Sleep(time.Duration(n) * time.Millisecond) -} - -func longRunningFunc() { - for { - runtime.Gosched() - } -} - -func longRunningPoolFunc(arg interface{}) { - if ch, ok := arg.(chan struct{}); ok { - <-ch - return - } - for { - runtime.Gosched() - } -} - -func BenchmarkGoroutines(b *testing.B) { - var wg sync.WaitGroup - for i := 0; i < b.N; i++ { - wg.Add(RunTimes) - for j := 0; j < RunTimes; j++ { - go func() { - demo(nil, nil) - wg.Done() - }() - } - wg.Wait() - } -} - -func BenchmarkSemaphore(b *testing.B) { - var wg sync.WaitGroup - sema := make(chan struct{}, BenchAntsSize) - - for i := 0; i < b.N; i++ { - wg.Add(RunTimes) - for j := 0; j < RunTimes; j++ { - sema <- struct{}{} - go func() { - demo(nil, nil) - <-sema - wg.Done() - }() - } - wg.Wait() - } -} - -func BenchmarkAntsPoolThroughput(b *testing.B) { - p, _ := NewPool(BenchAntsSize, WithExpiryDuration(DefaultExpiredTime)) - defer p.Release() - b.StartTimer() - for i := 0; i < b.N; i++ { - for j := 0; j < RunTimes; j++ { - _ = p.Submit(demoFunc) - } - } - b.StopTimer() -} diff --git a/core/pipeline/pools_test.go b/core/pipeline/pools_test.go deleted file mode 100755 index 2482ef5ac..000000000 --- a/core/pipeline/pools_test.go +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright (C) INFINI Labs & INFINI LIMITED. -// -// The INFINI Framework is offered under the GNU Affero General Public License v3.0 -// and as commercial software. -// -// For commercial licensing, contact us at: -// - Website: infinilabs.com -// - Email: hello@infini.ltd -// -// Open Source licensed under AGPL V3: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -// MIT License - -// Copyright (c) 2018 Andy Pan - -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -package pipeline - -import ( - "log" - "os" - "runtime" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -const ( - _ = 1 << (10 * iota) - KiB // 1024 - MiB // 1048576 - // GiB // 1073741824 - // TiB // 1099511627776 (超过了int32的范围) - // PiB // 1125899906842624 - // EiB // 1152921504606846976 - // ZiB // 1180591620717411303424 (超过了int64的范围) - // YiB // 1208925819614629174706176 -) - -const ( - Param = 100 - AntsSize = 1000 - TestSize = 10000 - n = 100000 -) - -var curMem uint64 - -// TestAntsPoolWithFuncWaitToGetWorker is used to test waiting to get worker. -func TestAntsPoolWithFuncWaitToGetWorker(t *testing.T) { - var wg sync.WaitGroup - p, _ := NewPoolWithFunc(AntsSize, func(i interface{}) { - demoPoolFunc(i) - wg.Done() - }) - defer p.Release() - - for i := 0; i < n; i++ { - wg.Add(1) - _ = p.Invoke(Param) - } - wg.Wait() - t.Logf("pool with func, running workers number:%d", p.Running()) - mem := runtime.MemStats{} - runtime.ReadMemStats(&mem) - curMem = mem.TotalAlloc/MiB - curMem - t.Logf("memory usage:%d MB", curMem) -} - -func TestAntsPoolWithFuncWaitToGetWorkerPreMalloc(t *testing.T) { - var wg sync.WaitGroup - p, _ := NewPoolWithFunc(AntsSize, func(i interface{}) { - demoPoolFunc(i) - wg.Done() - }, WithPreAlloc(true)) - defer p.Release() - - for i := 0; i < n; i++ { - wg.Add(1) - _ = p.Invoke(Param) - } - wg.Wait() - t.Logf("pool with func, running workers number:%d", p.Running()) - mem := runtime.MemStats{} - runtime.ReadMemStats(&mem) - curMem = mem.TotalAlloc/MiB - curMem - t.Logf("memory usage:%d MB", curMem) -} - -// TestAntsPoolGetWorkerFromCache is used to test getting worker from sync.Pool. -func TestAntsPoolGetWorkerFromCache(t *testing.T) { - p, _ := NewPool(TestSize) - defer p.Release() - - for i := 0; i < AntsSize; i++ { - _ = p.Submit(demoFunc) - } - time.Sleep(2 * DefaultCleanIntervalTime) - _ = p.Submit(demoFunc) - t.Logf("pool, running workers number:%d", p.Running()) - mem := runtime.MemStats{} - runtime.ReadMemStats(&mem) - curMem = mem.TotalAlloc/MiB - curMem - t.Logf("memory usage:%d MB", curMem) -} - -// TestAntsPoolWithFuncGetWorkerFromCache is used to test getting worker from sync.Pool. -func TestAntsPoolWithFuncGetWorkerFromCache(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - dur := 10 - p, _ := NewPoolWithFunc(TestSize, demoPoolFunc) - defer p.Release() - - for i := 0; i < AntsSize; i++ { - _ = p.Invoke(dur) - } - time.Sleep(2 * DefaultCleanIntervalTime) - _ = p.Invoke(dur) - t.Logf("pool with func, running workers number:%d", p.Running()) - mem := runtime.MemStats{} - runtime.ReadMemStats(&mem) - curMem = mem.TotalAlloc/MiB - curMem - t.Logf("memory usage:%d MB", curMem) -} - -func TestAntsPoolWithFuncGetWorkerFromCachePreMalloc(t *testing.T) { - dur := 10 - p, _ := NewPoolWithFunc(TestSize, demoPoolFunc, WithPreAlloc(true)) - defer p.Release() - - for i := 0; i < AntsSize; i++ { - _ = p.Invoke(dur) - } - time.Sleep(2 * DefaultCleanIntervalTime) - _ = p.Invoke(dur) - t.Logf("pool with func, running workers number:%d", p.Running()) - mem := runtime.MemStats{} - runtime.ReadMemStats(&mem) - curMem = mem.TotalAlloc/MiB - curMem - t.Logf("memory usage:%d MB", curMem) -} - -//------------------------------------------------------------------------------------------- -// Contrast between goroutines without a pool and goroutines with ants pool. -//------------------------------------------------------------------------------------------- - -func TestNoPool(t *testing.T) { - var wg sync.WaitGroup - for i := 0; i < n; i++ { - wg.Add(1) - go func() { - demo(nil, nil) - wg.Done() - }() - } - - wg.Wait() - mem := runtime.MemStats{} - runtime.ReadMemStats(&mem) - curMem = mem.TotalAlloc/MiB - curMem - t.Logf("memory usage:%d MB", curMem) -} - -func TestAntsPool(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - defer Release() - var wg sync.WaitGroup - for i := 0; i < n; i++ { - wg.Add(1) - _ = Submit(demoFunc) - } - wg.Wait() - - t.Logf("pool, capacity:%d", Cap()) - t.Logf("pool, running workers number:%d", Running()) - t.Logf("pool, free workers number:%d", Free()) - - mem := runtime.MemStats{} - runtime.ReadMemStats(&mem) - curMem = mem.TotalAlloc/MiB - curMem - t.Logf("memory usage:%d MB", curMem) -} - -func TestNonblockingSubmitWithFunc(t *testing.T) { - poolSize := 10 - var wg sync.WaitGroup - p, err := NewPoolWithFunc(poolSize, func(i interface{}) { - longRunningPoolFunc(i) - wg.Done() - }, WithNonblocking(true)) - assert.NoError(t, err, "create TimingPool failed: %v", err) - defer p.Release() - ch := make(chan struct{}) - wg.Add(poolSize) - for i := 0; i < poolSize-1; i++ { - assert.NoError(t, p.Invoke(ch), "nonblocking submit when pool is not full shouldn't return error") - } - // p is full now. - assert.NoError(t, p.Invoke(ch), "nonblocking submit when pool is not full shouldn't return error") - assert.EqualError(t, p.Invoke(nil), ErrPoolOverload.Error(), - "nonblocking submit when pool is full should get an ErrPoolOverload") - // interrupt f to get an available worker - close(ch) - wg.Wait() - assert.NoError(t, p.Invoke(nil), "nonblocking submit when pool is not full shouldn't return error") -} - -func TestInfinitePoolWithFunc(t *testing.T) { - c := make(chan struct{}) - p, _ := NewPoolWithFunc(-1, func(i interface{}) { - demoPoolFunc(i) - <-c - }) - _ = p.Invoke(10) - _ = p.Invoke(10) - c <- struct{}{} - c <- struct{}{} - if n := p.Running(); n != 2 { - t.Errorf("expect 2 workers running, but got %d", n) - } - if n := p.Free(); n != -1 { - t.Errorf("expect -1 of free workers by unlimited pool, but got %d", n) - } - p.Tune(10) - if capacity := p.Cap(); capacity != -1 { - t.Fatalf("expect capacity: -1 but got %d", capacity) - } - var err error - _, err = NewPoolWithFunc(-1, demoPoolFunc, WithPreAlloc(true)) - if err != ErrInvalidPreAllocSize { - t.Errorf("expect ErrInvalidPreAllocSize but got %v", err) - } -} - -func TestReleaseWhenRunningPoolWithFunc(t *testing.T) { - var wg sync.WaitGroup - p, _ := NewPoolWithFunc(1, func(i interface{}) { - t.Log("do task", i) - time.Sleep(1 * time.Second) - }) - wg.Add(2) - go func() { - t.Log("start aaa") - defer func() { - wg.Done() - t.Log("stop aaa") - }() - for i := 0; i < 30; i++ { - _ = p.Invoke(i) - } - }() - - go func() { - t.Log("start bbb") - defer func() { - wg.Done() - t.Log("stop bbb") - }() - for i := 100; i < 130; i++ { - _ = p.Invoke(i) - } - }() - - time.Sleep(3 * time.Second) - p.Release() - t.Log("wait for all goroutines to exit...") - wg.Wait() -} - -func TestRestCodeCoverage(t *testing.T) { - _, err := NewPool(-1, WithExpiryDuration(-1)) - t.Log(err) - _, err = NewPool(1, WithExpiryDuration(-1)) - t.Log(err) - _, err = NewPoolWithFunc(-1, demoPoolFunc, WithExpiryDuration(-1)) - t.Log(err) - _, err = NewPoolWithFunc(1, demoPoolFunc, WithExpiryDuration(-1)) - t.Log(err) - - options := Options{} - options.ExpiryDuration = time.Duration(10) * time.Second - options.Nonblocking = true - options.PreAlloc = true - poolOpts, _ := NewPool(1, WithOptions(options)) - t.Logf("Pool with options, capacity: %d", poolOpts.Cap()) - - p0, _ := NewPool(TestSize, WithLogger(log.New(os.Stderr, "", log.LstdFlags))) - defer func() { - _ = p0.Submit(demoFunc) - }() - defer p0.Release() - for i := 0; i < n; i++ { - _ = p0.Submit(demoFunc) - } - t.Logf("pool, capacity:%d", p0.Cap()) - t.Logf("pool, running workers number:%d", p0.Running()) - t.Logf("pool, free workers number:%d", p0.Free()) - p0.Tune(TestSize) - p0.Tune(TestSize / 10) - t.Logf("pool, after tuning capacity, capacity:%d, running:%d", p0.Cap(), p0.Running()) - - pprem, _ := NewPool(TestSize, WithPreAlloc(true)) - defer func() { - _ = pprem.Submit(demoFunc) - }() - defer pprem.Release() - for i := 0; i < n; i++ { - _ = pprem.Submit(demoFunc) - } - t.Logf("pre-malloc pool, capacity:%d", pprem.Cap()) - t.Logf("pre-malloc pool, running workers number:%d", pprem.Running()) - t.Logf("pre-malloc pool, free workers number:%d", pprem.Free()) - pprem.Tune(TestSize) - pprem.Tune(TestSize / 10) - t.Logf("pre-malloc pool, after tuning capacity, capacity:%d, running:%d", pprem.Cap(), pprem.Running()) - - p, _ := NewPoolWithFunc(TestSize, demoPoolFunc) - defer func() { - _ = p.Invoke(Param) - }() - defer p.Release() - for i := 0; i < n; i++ { - _ = p.Invoke(Param) - } - time.Sleep(DefaultCleanIntervalTime) - t.Logf("pool with func, capacity:%d", p.Cap()) - t.Logf("pool with func, running workers number:%d", p.Running()) - t.Logf("pool with func, free workers number:%d", p.Free()) - p.Tune(TestSize) - p.Tune(TestSize / 10) - t.Logf("pool with func, after tuning capacity, capacity:%d, running:%d", p.Cap(), p.Running()) - - ppremWithFunc, _ := NewPoolWithFunc(TestSize, demoPoolFunc, WithPreAlloc(true)) - defer func() { - _ = ppremWithFunc.Invoke(Param) - }() - defer ppremWithFunc.Release() - for i := 0; i < n; i++ { - _ = ppremWithFunc.Invoke(Param) - } - time.Sleep(DefaultCleanIntervalTime) - t.Logf("pre-malloc pool with func, capacity:%d", ppremWithFunc.Cap()) - t.Logf("pre-malloc pool with func, running workers number:%d", ppremWithFunc.Running()) - t.Logf("pre-malloc pool with func, free workers number:%d", ppremWithFunc.Free()) - ppremWithFunc.Tune(TestSize) - ppremWithFunc.Tune(TestSize / 10) - t.Logf("pre-malloc pool with func, after tuning capacity, capacity:%d, running:%d", ppremWithFunc.Cap(), - ppremWithFunc.Running()) -} diff --git a/core/rotate/rotate_test.go b/core/rotate/rotate_test.go index c9706bec9..403e8d29c 100755 --- a/core/rotate/rotate_test.go +++ b/core/rotate/rotate_test.go @@ -25,25 +25,3 @@ // +build linux package rotate - -import ( - "log" - "os" - "os/signal" - "syscall" -) - -// Example of how to rotate in response to SIGHUP. -func ExampleLogger_Rotate() { - l := &RotateWriter{} - log.SetOutput(l) - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGHUP) - - go func() { - for { - <-c - l.Rotate() - } - }() -} diff --git a/core/rotate/writer_test.go b/core/rotate/writer_test.go index 6204995d7..66f5c4e21 100755 --- a/core/rotate/writer_test.go +++ b/core/rotate/writer_test.go @@ -155,46 +155,6 @@ func TestDefaultFilename(t *testing.T) { existsWithContent(filename, b, t) } -func TestAutoRotate(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - megabyte = 1 - - dir := makeTempDir("TestAutoRotate", t) - defer os.RemoveAll(dir) - - filename := logFile(dir) - l := &RotateWriter{ - Filename: filename, - MaxFileSize: 10, - } - defer l.Close() - b := []byte("boo!") - n, err := l.Write(b) - isNil(err, t) - equals(len(b), n, t) - - existsWithContent(filename, b, t) - fileCount(dir, 1, t) - - newFakeTime() - - b2 := []byte("foooooo!") - n, err = l.Write(b2) - isNil(err, t) - equals(len(b2), n, t) - - // the old logfile should be moved aside and the main logfile should have - // only the last write in it. - existsWithContent(filename, b2, t) - - // the backup file will use the current fake time and have the old contents. - existsWithContent(backupFile(dir), b, t) - - fileCount(dir, 2, t) -} - func TestFirstWriteRotate(t *testing.T) { megabyte = 1 dir := makeTempDir("TestFirstWriteRotate", t) @@ -225,133 +185,6 @@ func TestFirstWriteRotate(t *testing.T) { fileCount(dir, 2, t) } -func TestMaxBackups(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - megabyte = 1 - dir := makeTempDir("TestMaxBackups", t) - defer os.RemoveAll(dir) - - filename := logFile(dir) - l := &RotateWriter{ - Filename: filename, - MaxFileSize: 10, - MaxRotationCount: 1, - } - defer l.Close() - b := []byte("boo!") - n, err := l.Write(b) - isNil(err, t) - equals(len(b), n, t) - - existsWithContent(filename, b, t) - fileCount(dir, 1, t) - - newFakeTime() - - // this will put us over the max - b2 := []byte("foooooo!") - n, err = l.Write(b2) - isNil(err, t) - equals(len(b2), n, t) - - // this will use the new fake time - secondFilename := backupFile(dir) - existsWithContent(secondFilename, b, t) - - // make sure the old file still exists with the same content. - existsWithContent(filename, b2, t) - - fileCount(dir, 2, t) - - newFakeTime() - - // this will make us rotate again - b3 := []byte("baaaaaar!") - n, err = l.Write(b3) - isNil(err, t) - equals(len(b3), n, t) - - // this will use the new fake time - thirdFilename := backupFile(dir) - existsWithContent(thirdFilename, b2, t) - - existsWithContent(filename, b3, t) - - // we need to wait a little bit since the files get deleted on a different - // goroutine. - <-time.After(time.Millisecond * 10) - - // should only have two files in the dir still - fileCount(dir, 2, t) - - // second file name should still exist - existsWithContent(thirdFilename, b2, t) - - // should have deleted the first backup - notExist(secondFilename, t) - - // now test that we don't delete directories or non-logfile files - - newFakeTime() - - // create a file that is close to but different from the logfile name. - // It shouldn't get caught by our deletion filters. - notlogfile := logFile(dir) + ".foo" - err = ioutil.WriteFile(notlogfile, []byte("data"), 0644) - isNil(err, t) - - // Make a directory that exactly matches our log file filters... it still - // shouldn't get caught by the deletion filter since it's a directory. - notlogfiledir := backupFile(dir) - err = os.Mkdir(notlogfiledir, 0700) - isNil(err, t) - - newFakeTime() - - // this will use the new fake time - fourthFilename := backupFile(dir) - - // Create a log file that is/was being compressed - this should - // not be counted since both the compressed and the uncompressed - // log files still exist. - compLogFile := fourthFilename + compressSuffix - err = ioutil.WriteFile(compLogFile, []byte("compress"), 0644) - isNil(err, t) - - // this will make us rotate again - b4 := []byte("baaaaaaz!") - n, err = l.Write(b4) - isNil(err, t) - equals(len(b4), n, t) - - existsWithContent(fourthFilename, b3, t) - existsWithContent(fourthFilename+compressSuffix, []byte("compress"), t) - - // we need to wait a little bit since the files get deleted on a different - // goroutine. - <-time.After(time.Millisecond * 10) - - // We should have four things in the directory now - the 2 log files, the - // not log file, and the directory - fileCount(dir, 5, t) - - // third file name should still exist - existsWithContent(filename, b4, t) - - existsWithContent(fourthFilename, b3, t) - - // should have deleted the first filename - notExist(thirdFilename, t) - - // the not-a-logfile should still exist - exists(notlogfile, t) - - // the directory - exists(notlogfiledir, t) -} - func TestCleanupExistingBackups(t *testing.T) { // test that if we start with more backup files than we're supposed to have // in total, that extra ones get cleaned up when we rotate. @@ -407,75 +240,6 @@ func TestCleanupExistingBackups(t *testing.T) { fileCount(dir, 2, t) } -func TestMaxAge(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - megabyte = 1 - - dir := makeTempDir("TestMaxAge", t) - defer os.RemoveAll(dir) - - filename := logFile(dir) - l := &RotateWriter{ - Filename: filename, - MaxFileSize: 10, - MaxFileAge: 1, - } - defer l.Close() - b := []byte("boo!") - n, err := l.Write(b) - isNil(err, t) - equals(len(b), n, t) - - existsWithContent(filename, b, t) - fileCount(dir, 1, t) - - // two days later - newFakeTime() - - b2 := []byte("foooooo!") - n, err = l.Write(b2) - isNil(err, t) - equals(len(b2), n, t) - existsWithContent(backupFile(dir), b, t) - - // we need to wait a little bit since the files get deleted on a different - // goroutine. - <-time.After(10 * time.Millisecond) - - // We should still have 2 log files, since the most recent backup was just - // created. - fileCount(dir, 2, t) - - existsWithContent(filename, b2, t) - - // we should have deleted the old file due to being too old - existsWithContent(backupFile(dir), b, t) - - // two days later - newFakeTime() - - b3 := []byte("baaaaar!") - n, err = l.Write(b3) - isNil(err, t) - equals(len(b3), n, t) - existsWithContent(backupFile(dir), b2, t) - - // we need to wait a little bit since the files get deleted on a different - // goroutine. - <-time.After(10 * time.Millisecond) - - // We should have 2 log files - the main log file, and the most recent - // backup. The earlier backup is past the cutoff and should be gone. - fileCount(dir, 2, t) - - existsWithContent(filename, b3, t) - - // we should have deleted the old file due to being too old - existsWithContent(backupFile(dir), b2, t) -} - func TestOldLogFiles(t *testing.T) { megabyte = 1 @@ -537,145 +301,6 @@ func TestTimeFromName(t *testing.T) { } } -func TestLocalTime(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - megabyte = 1 - - dir := makeTempDir("TestLocalTime", t) - defer os.RemoveAll(dir) - - l := &RotateWriter{ - Filename: logFile(dir), - MaxFileSize: 10, - LocalTime: true, - } - defer l.Close() - b := []byte("boo!") - n, err := l.Write(b) - isNil(err, t) - equals(len(b), n, t) - - b2 := []byte("fooooooo!") - n2, err := l.Write(b2) - isNil(err, t) - equals(len(b2), n2, t) - - existsWithContent(logFile(dir), b2, t) - existsWithContent(backupFileLocal(dir), b, t) -} - -func TestRotate(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - dir := makeTempDir("TestRotate", t) - defer os.RemoveAll(dir) - - filename := logFile(dir) - - l := &RotateWriter{ - Filename: filename, - MaxRotationCount: 1, - MaxFileSize: 100, // megabytes - } - defer l.Close() - b := []byte("boo!") - n, err := l.Write(b) - isNil(err, t) - equals(len(b), n, t) - - existsWithContent(filename, b, t) - fileCount(dir, 1, t) - - newFakeTime() - - err = l.Rotate() - isNil(err, t) - - // we need to wait a little bit since the files get deleted on a different - // goroutine. - <-time.After(10 * time.Millisecond) - - filename2 := backupFile(dir) - existsWithContent(filename2, b, t) - existsWithContent(filename, []byte{}, t) - fileCount(dir, 2, t) - newFakeTime() - - err = l.Rotate() - isNil(err, t) - - // we need to wait a little bit since the files get deleted on a different - // goroutine. - <-time.After(10 * time.Millisecond) - - filename3 := backupFile(dir) - existsWithContent(filename3, []byte{}, t) - existsWithContent(filename, []byte{}, t) - fileCount(dir, 2, t) - - b2 := []byte("foooooo!") - n, err = l.Write(b2) - isNil(err, t) - equals(len(b2), n, t) - - // this will use the new fake time - existsWithContent(filename, b2, t) -} - -func TestCompressOnRotate(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - megabyte = 1 - - dir := makeTempDir("TestCompressOnRotate", t) - defer os.RemoveAll(dir) - - filename := logFile(dir) - l := &RotateWriter{ - Compress: true, - Filename: filename, - MaxFileSize: 10, - } - defer l.Close() - b := []byte("boo!") - n, err := l.Write(b) - isNil(err, t) - equals(len(b), n, t) - - existsWithContent(filename, b, t) - fileCount(dir, 1, t) - - newFakeTime() - - err = l.Rotate() - isNil(err, t) - - // the old logfile should be moved aside and the main logfile should have - // nothing in it. - existsWithContent(filename, []byte{}, t) - - // we need to wait a little bit since the files get compressed on a different - // goroutine. - <-time.After(300 * time.Millisecond) - - // a compressed version of the log file should now exist and the original - // should have been removed. - bc := new(bytes.Buffer) - gz := gzip.NewWriter(bc) - _, err = gz.Write(b) - isNil(err, t) - err = gz.Close() - isNil(err, t) - existsWithContent(backupFile(dir)+compressSuffix, bc.Bytes(), t) - notExist(backupFile(dir), t) - - fileCount(dir, 2, t) -} - func TestCompressOnResume(t *testing.T) { megabyte = 1 diff --git a/core/task/chrono/task_test.go b/core/task/chrono/task_test.go index 26adbadf3..44f600b96 100755 --- a/core/task/chrono/task_test.go +++ b/core/task/chrono/task_test.go @@ -26,7 +26,6 @@ package chrono import ( "context" "errors" - "os" "testing" "time" @@ -74,20 +73,6 @@ func TestNewScheduledRunnableTask(t *testing.T) { assert.Error(t, err) } -func TestNewTriggerTaskWithTimezone(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - - trigger, err := CreateCronTrigger("CRON_TZ=America/New_York 0 9 18 * * 1", time.Local) - assert.Nil(t, err) - loc, _ := time.LoadLocation("America/New_York") - assert.Equal(t, loc, trigger.location) - ctx := NewSimpleTriggerContext() - tm := trigger.NextExecutionTime(ctx) - assert.Equal(t, 7, tm.Hour()) -} - func TestNewTriggerTask(t *testing.T) { trigger, err := CreateCronTrigger("* * * * * *", time.Local) assert.Nil(t, err) diff --git a/dev.go b/dev.go index a11a95b6f..a87e0038d 100644 --- a/dev.go +++ b/dev.go @@ -34,14 +34,15 @@ import ( "expvar" "flag" "fmt" - "github.com/arl/statsviz" - log "github.com/cihub/seelog" - "infini.sh/framework/core/global" "net/http" _ "net/http/pprof" "os" "runtime" "runtime/pprof" + + "github.com/arl/statsviz" + "infini.sh/framework/core/global" + "infini.sh/framework/core/log" ) var cpuproFile string diff --git a/docs/content.en/docs/development/create_new_application.md b/docs/content.en/docs/development/create_new_application.md index 946921ab3..f67832299 100644 --- a/docs/content.en/docs/development/create_new_application.md +++ b/docs/content.en/docs/development/create_new_application.md @@ -77,7 +77,6 @@ APP_STATIC_FOLDER := .public APP_STATIC_PACKAGE := public APP_UI_FOLDER := ui APP_PLUGIN_FOLDER := plugins -PREFER_MANAGED_VENDOR=fase include ../framework/Makefile ``` @@ -88,18 +87,10 @@ include ../framework/Makefile building new_app 1.0.0_SNAPSHOT main /Users/medcl/go/src/infini.sh/new_app framework path: /Users/medcl/go/src/infini.sh/framework -fatal: not a git repository (or any of the parent directories): .git update generated info update configs -(cd ../framework/ && make update-plugins) || true # build plugins in framework -GOPATH=~/go:~/go/src/infini.sh/framework/../vendor/ CGO_ENABLED=0 GRPC_GO_REQUIRE_HANDSHAKE=off GO15VENDOREXPERIMENT="1" GO111MODULE=off go build -a -gcflags=all="-l -B" -ldflags '-static' -ldflags='-s -w' -gcflags "-m" --work -o /Users/medcl/go/src/infini.sh/new_app/bin/new_app +CGO_ENABLED=0 go build -a -gcflags=all="-l -B" -ldflags '-static' -ldflags='-s -w' -gcflags "-m" --work -o /Users/medcl/go/src/infini.sh/new_app/bin/new_app WORK=/var/folders/j5/qd4qt3n55dz053d93q2mswfr0000gn/T/go-build435280758 -# infini.sh/new_app -./main.go:17:9: can inline main.deferwrap1 -./main.go:21:12: can inline main.func2 -./main.go:18:22: func literal does not escape -./main.go:19:45: &api.APIModule{} escapes to heap -./main.go:21:12: func literal escapes to heap restore generated info ``` diff --git a/docs/content.en/docs/development/setup_golang_environment.md b/docs/content.en/docs/development/setup_golang_environment.md index ad6421024..d03aa6f2e 100644 --- a/docs/content.en/docs/development/setup_golang_environment.md +++ b/docs/content.en/docs/development/setup_golang_environment.md @@ -9,10 +9,10 @@ Refer the official guide to install Golang: [https://go.dev/doc/install](https:/ ## Golang Version -Verify your Go version: +Verify your Go version (1.21+ required): ```bash -➜ loadgen git:(master) ✗ go version +➜ ~ go version go version go1.23.3 darwin/arm64 ``` @@ -30,34 +30,42 @@ mkdir -p infini.sh/ ## Cloning Dependencies -Clone the required dependency repositories: +Clone the framework repository: ```bash cd ~/go/src/infini.sh -git@github.com:infinilabs/framework.git +git clone git@github.com:infinilabs/framework.git ``` +> Note: No separate vendor repository is needed. All dependencies are managed via Go modules. + ## Cloning Application Code For example, to work with the Loadgen project: ```bash cd ~/go/src/infini.sh -git@github.com:infinilabs/loadgen.git +git clone git@github.com:infinilabs/loadgen.git ``` ## Building the Project Build the project using the Makefile: ```bash cd loadgen -make +make build ``` -The make command will automatically download the required dependency repositories. ## Customization Build with Built-in Environments -For example, if you want to expose more debug-level information, such as detecting data races, you can compile a debug build. You may also specify the `GOPATH` as needed. Use the following command: +For example, if you want to expose more debug-level information, such as detecting data races, you can compile a debug build: + +```bash +DEV=true make build +``` + +You can also specify a custom `GOPATH` if needed: ```bash -DEV=true GOPATH="/Users//go" make build +GOPATH="/Users//go" make build ``` + To learn more about the Makefile and its commands, refer to this [Reference](../references/makefile.md). diff --git a/docs/content.en/docs/references/makefile.md b/docs/content.en/docs/references/makefile.md index 515316047..906e8d133 100644 --- a/docs/content.en/docs/references/makefile.md +++ b/docs/content.en/docs/references/makefile.md @@ -37,14 +37,16 @@ This approach ensures better maintainability and faster setup for new projects. To build the `Loadgen` application using the framework, you can run the following command: ```shell -➜ loadgen git:(main) DEV=false OFFLINE_BUILD=true make build +➜ loadgen git:(main) OFFLINE_BUILD=true make build ``` Explanation of the Command: -- DEV=false: Sets the development mode to false, indicating a production build. -- OFFLINE_BUILD=true: Enables offline build mode, ensuring the build process avoids fetching resources from external sources. -- make build: Invokes the build target defined in the framework’s Makefile, compiling the application according to the specified settings. +- OFFLINE_BUILD=true: Enables offline build mode, skipping the `git pull` step for the framework repository. +- make build: Invokes the build target defined in the framework's Makefile, compiling the application. -This example demonstrates how you can customize the build process using environment variables while leveraging the reusable commands provided by the framework. +For development builds with race detection and extra debug info: +```shell +➜ loadgen git:(main) DEV=true make build +``` ## Commands @@ -67,10 +69,10 @@ This example demonstrates how you can customize the build process using environm | `build-bsd` | Builds the application binary for BSD systems | Supports FreeBSD, NetBSD, and OpenBSD | | `all` | Cleans, configures, and builds binaries for all supported platforms | | | `all-platform` | Builds binaries for all platforms, including BSD, Linux, macOS, and Windows | | -| `format` | Formats all Go files excluding vendor directory | Uses `go fmt` | +| `format` | Formats all Go source files | Uses `go fmt` | | `clean_data` | Removes data and logs directories | | | `clean` | Cleans all build artifacts and resets the output directory | Depends on `clean_data` | -| `init` | Initializes the build environment | Checks/clones framework repositories | +| `init` | Initializes the build environment | Checks/clones framework repository | --- @@ -89,25 +91,21 @@ This example demonstrates how you can customize the build process using environm | `APP_PLUGIN_PKG` | Plugins package name | `$(APP_PLUGIN_FOLDER)` | | `APP_NEED_CGO` | Determines if CGO is required (0 = disabled, 1 = enabled) | `0` | | `VERSION` | Release version from the environment | | -| `GOPATH` | Go workspace path | `~/go` | +| `GOPATH` | Go workspace path. Can be overridden to use a custom location. | `~/go` | | `BUILD_NUMBER` | Build number | `001` | -| `DEV` | Enables or disables development mode. Set to true for development builds, false for production builds. | `false` | -| `OFFLINE_BUILD` | Enables offline build mode, preventing the download of external resources during the build process. | `false` | -| `GO` | Go environment settings | `GO15VENDOREXPERIMENT="1" GO111MODULE=off go` | +| `DEV` | Enables development mode (adds `-tags dev` to build). Set to any non-empty value to enable. | | +| `OFFLINE_BUILD` | Skips `git pull` for the framework during `init`. Set to any non-empty value to enable. | | +| `GO` | Go command | `go` | | `FRAMEWORK_FOLDER` | Path to INFINI Framework folder | `$(INFINI_BASE_FOLDER)/framework` | | `FRAMEWORK_REPO` | Framework repository URL | `https://github.com/infinilabs/framework.git` | | `FRAMEWORK_BRANCH` | Git branch for the framework | `main` | -| `FRAMEWORK_VENDOR_FOLDER` | Path to framework vendor folder | `$(FRAMEWORK_FOLDER)/../vendor/` | -| `FRAMEWORK_VENDOR_REPO` | Vendor repository URL | `https://github.com/infinilabs/framework-vendor.git` | -| `FRAMEWORK_VENDOR_BRANCH` | Vendor repository branch | `main` | -| `PREFER_MANAGED_VENDOR` | Determines whether to use a managed vendor directory or fetch dependencies dynamically. If set to `1`, the build process will prioritize the pre-downloaded vendor folder (`FRAMEWORK_VENDOR_FOLDER`). If set to `0`, dependencies will be fetched from the `FRAMEWORK_VENDOR_REPO`. | `1` | --- ### Notes - - **Framework Dependencies**: This `Makefile` integrates with INFINI Framework, requiring external repositories for the framework and vendor files. Ensure these are cloned and accessible. + - **Go Modules**: All dependencies are managed via Go modules (`go.mod`). No external vendor repository is required. - **Cross-Platform Builds**: Targets like `build-linux` and `build-darwin` compile binaries for multiple architectures, ensuring compatibility across platforms. - **Plugin Updates**: Plugins are dynamically discovered and updated using a tool within the framework. Ensure `plugin-discovery` exists and is built. - **Environment Variables**: Many configurations (e.g., `GOPATH`, `VERSION`, `EOL`) can be overridden via environment variables for flexibility. diff --git a/go.mod b/go.mod index 65ac0c52e..5a4ee6646 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,8 @@ module infini.sh/framework -go 1.23.3 +go 1.25.0 -replace github.com/libdns/libdns => ../vendor/src/github.com/libdns/libdns - -replace github.com/libdns/tencentcloud => ../vendor/src/github.com/libdns/tencentcloud - -replace github.com/caddyserver/certmagic => ../vendor/src/github.com/caddyserver/certmagic - -replace github.com/caddyserver/zerossl => ../vendor/src/github.com/caddyserver/zerossl - -replace github.com/quipo/statsd => ../vendor/src/github.com/quipo/statsd - -replace github.com/cihub/seelog => ../vendor/src/github.com/cihub/seelog - -replace github.com/gopkg.in/gomail.v2 => ../vendor/src/github.com/gopkg.in/gomail.v2 +replace github.com/cihub/seelog => ./lib/seelog require ( github.com/OneOfOne/xxhash v1.2.8 @@ -23,14 +11,14 @@ require ( github.com/arl/statsviz v0.6.0 github.com/bkaradzic/go-lz4 v1.0.0 github.com/buger/jsonparser v1.1.1 - github.com/caddyserver/certmagic v0.23.0 + github.com/caddyserver/certmagic v0.25.3 github.com/cihub/seelog v0.0.0-00010101000000-000000000000 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/dgraph-io/badger/v4 v4.7.0 github.com/dgraph-io/ristretto v0.2.0 github.com/emirpasic/gods v1.18.1 github.com/fsnotify/fsnotify v1.9.0 - github.com/go-ldap/ldap/v3 v3.4.11 + github.com/go-ldap/ldap/v3 v3.4.13 github.com/go-redis/redis/v8 v8.11.5 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang-jwt/jwt/v4 v4.5.2 @@ -38,7 +26,6 @@ require ( github.com/google/go-cmp v0.7.0 github.com/google/go-github v17.0.0+incompatible github.com/gookit/validate v1.5.6 - github.com/gopkg.in/gomail.v2 v0.0.0-00010101000000-000000000000 github.com/gorilla/context v1.1.2 github.com/gorilla/sessions v1.4.0 github.com/gorilla/websocket v1.5.3 @@ -47,14 +34,12 @@ require ( github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/kardianos/service v1.2.2 github.com/klauspost/compress v1.18.0 - github.com/libdns/tencentcloud v1.2.1 github.com/magiconair/properties v1.8.10 github.com/mailru/easyjson v0.9.0 github.com/minio/minio-go/v7 v7.0.90 github.com/mitchellh/mapstructure v1.5.0 github.com/nsqio/nsq v1.3.0 github.com/pkg/errors v0.9.1 - github.com/quipo/statsd v0.0.0-00010101000000-000000000000 github.com/r3labs/diff/v2 v2.15.1 github.com/rs/cors v1.11.1 github.com/rs/xid v1.6.0 @@ -71,14 +56,14 @@ require ( github.com/twmb/franz-go/pkg/kmsg v1.11.2 github.com/valyala/tcplisten v1.0.0 github.com/zeebo/sbloom v0.0.0-20151106181526-405c65bd9be0 - go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.37.0 - golang.org/x/net v0.39.0 + go.uber.org/zap v1.27.1 + golang.org/x/crypto v0.50.0 + golang.org/x/net v0.53.0 golang.org/x/oauth2 v0.29.0 - golang.org/x/sys v0.32.0 - golang.org/x/text v0.24.0 + golang.org/x/sys v0.43.0 + golang.org/x/text v0.36.0 golang.org/x/time v0.11.0 - golang.org/x/tools v0.32.0 + golang.org/x/tools v0.44.0 google.golang.org/grpc v1.71.1 gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/hjson/hjson-go.v3 v3.3.0 @@ -90,9 +75,9 @@ require ( ) require ( - github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/Azure/go-ntlmssp v0.1.0 // indirect github.com/bits-and-blooms/bitset v1.12.0 // indirect - github.com/caddyserver/zerossl v0.1.3 // indirect + github.com/caddyserver/zerossl v0.1.5 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect @@ -108,6 +93,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -118,13 +104,13 @@ require ( github.com/gorilla/securecookie v1.1.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.10 // indirect - github.com/libdns/libdns v1.0.0 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/libdns/libdns v1.1.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect - github.com/mholt/acmez/v3 v3.1.2 // indirect - github.com/miekg/dns v1.1.63 // indirect + github.com/mholt/acmez/v3 v3.1.6 // indirect + github.com/miekg/dns v1.1.72 // indirect github.com/minio/crc64nvme v1.0.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -157,9 +143,9 @@ require ( go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap/exp v0.3.0 // indirect - golang.org/x/mod v0.24.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/term v0.31.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/term v0.42.0 // indirect google.golang.org/appengine v1.6.6 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect google.golang.org/protobuf v1.36.6 // indirect diff --git a/go.sum b/go.sum index ce5b6c490..3d3f64898 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= +github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= @@ -12,6 +14,7 @@ github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/arl/statsviz v0.6.0 h1:jbW1QJkEYQkufd//4NDYRSNBpwJNrdzPahF7ZmoGdyE= @@ -23,6 +26,10 @@ github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwj github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/caddyserver/certmagic v0.25.3 h1:mGf5ba8F7xA4c5jfDZZbK2buY1VEkbnwpMDixaju94A= +github.com/caddyserver/certmagic v0.25.3/go.mod h1:YVs43D5+H/Dckt4bTga1KSO/xYfFBfVZainGDywYPAA= +github.com/caddyserver/zerossl v0.1.5 h1:dkvOjBAEEtY6LIGAHei7sw2UgqSD6TrWweXpV7lvEvE= +github.com/caddyserver/zerossl v0.1.5/go.mod h1:CxA0acn7oEGO6//4rtrRjYgEoa4MFw/XofZnrYwGqG4= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -69,6 +76,8 @@ github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-ldap/ldap/v3 v3.2.4/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg= github.com/go-ldap/ldap/v3 v3.4.11 h1:4k0Yxweg+a3OyBLjdYn5OKglv18JNvfDykSoI8bW0gU= github.com/go-ldap/ldap/v3 v3.4.11/go.mod h1:bY7t0FLK8OAVpp/vV6sSlpz3EQDGcQwc8pF0ujLgKvM= +github.com/go-ldap/ldap/v3 v3.4.13 h1:+x1nG9h+MZN7h/lUi5Q3UZ0fJ1GyDQYbPvbuH38baDQ= +github.com/go-ldap/ldap/v3 v3.4.13/go.mod h1:LxsGZV6vbaK0sIvYfsv47rfh4ca0JXokCoKjZxsszv0= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -185,6 +194,8 @@ github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYW github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -193,6 +204,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/libdns/libdns v1.1.1 h1:wPrHrXILoSHKWJKGd0EiAVmiJbFShguILTg9leS/P/U= +github.com/libdns/libdns v1.1.1/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.7.4-0.20170902060319-8d7837e64d3c/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -212,8 +225,12 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mholt/acmez/v3 v3.1.2 h1:auob8J/0FhmdClQicvJvuDavgd5ezwLBfKuYmynhYzc= github.com/mholt/acmez/v3 v3.1.2/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= +github.com/mholt/acmez/v3 v3.1.6 h1:eGVQNObP0pBN4sxqrXeg7MYqTOWyoiYpQqITVWlrevk= +github.com/mholt/acmez/v3 v3.1.6/go.mod h1:5nTPosTGosLxF3+LU4ygbgMRFDhbAVpqMI4+a4aHLBY= github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -370,6 +387,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -378,10 +397,14 @@ golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -392,6 +415,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= @@ -402,6 +427,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -416,14 +443,20 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= @@ -435,6 +468,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -447,8 +482,6 @@ google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoA google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= -google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= diff --git a/lib/bytebufferpool/bytebuffer_test.go b/lib/bytebufferpool/bytebuffer_test.go index e95355933..45ff452f5 100644 --- a/lib/bytebufferpool/bytebuffer_test.go +++ b/lib/bytebufferpool/bytebuffer_test.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - "os" "testing" "time" @@ -165,15 +164,3 @@ func TestByteBufferGetStringConcurrent(t *testing.T) { } } } - -func TestByteBufferWriteSize(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - expectedS := "foobarbaz" - bb := ByteBuffer{} - for i := 0; i < 100; i++ { - bb.Write([]byte(expectedS)) - t.Log(i, ",", bb.Len(), ",", bb.Cap()) - } -} diff --git a/lib/bytebufferpool/pool_test.go b/lib/bytebufferpool/pool_test.go index ed0a8d9ea..14e80f55d 100644 --- a/lib/bytebufferpool/pool_test.go +++ b/lib/bytebufferpool/pool_test.go @@ -1,11 +1,8 @@ package bytebufferpool import ( - "os" "testing" "time" - - "github.com/stretchr/testify/assert" ) func TestPoolVariousSizesSerial(t *testing.T) { @@ -60,18 +57,3 @@ func allocNBytes(dst []byte, n int) []byte { } return append(dst, make([]byte, diff)...) } - -func TestCalibrate(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - - p := getPoolByTag("test") - for i := 0; i < 1000; i++ { - x := p.Get() - x.GrowTo(i) - } - t.Log(p.poolItems) - p.calibrate() - assert.Equal(t, p.maxItemSize, uint32(999)) -} diff --git a/lib/cache/cache_test.go b/lib/cache/cache_test.go index 1f5e7119f..e8e0d4aec 100644 --- a/lib/cache/cache_test.go +++ b/lib/cache/cache_test.go @@ -1,7 +1,6 @@ package ccache import ( - "os" "strconv" "sync/atomic" "testing" @@ -237,19 +236,12 @@ func Test_Cache_ReplaceChangesSize(t *testing.T) { } func Test_Cache_ResizeOnTheFly(t *testing.T) { - // On a busy system or during a slow run, the cleanup might take longer. - // When this happens, the test continues - // and runs its assertions (e.g., assert.Equal(t, cache.GetDropped(), 2)) - // before the cache has actually been pruned, causing the test to fail. - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } cache := New(Configure().MaxSize(9).ItemsToPrune(1)) for i := 0; i < 5; i++ { cache.Set(strconv.Itoa(i), i, time.Minute) } cache.SetMaxSize(3) - time.Sleep(time.Millisecond * 10) + time.Sleep(time.Millisecond * 100) assert.Equal(t, cache.GetDropped(), 2) assert.Nil(t, cache.Get("0")) assert.Nil(t, cache.Get("1")) @@ -258,7 +250,7 @@ func Test_Cache_ResizeOnTheFly(t *testing.T) { assert.Equal(t, cache.Get("4").Value(), 4) cache.Set("5", 5, time.Minute) - time.Sleep(time.Millisecond * 5) + time.Sleep(time.Millisecond * 100) assert.Equal(t, cache.GetDropped(), 1) assert.Nil(t, cache.Get("2")) assert.Equal(t, cache.Get("3").Value(), 3) @@ -267,7 +259,7 @@ func Test_Cache_ResizeOnTheFly(t *testing.T) { cache.SetMaxSize(10) cache.Set("6", 6, time.Minute) - time.Sleep(time.Millisecond * 10) + time.Sleep(time.Millisecond * 100) assert.Equal(t, cache.GetDropped(), 0) assert.Equal(t, cache.Get("3").Value(), 3) assert.Equal(t, cache.Get("4").Value(), 4) diff --git a/lib/cache/layeredcache_test.go b/lib/cache/layeredcache_test.go index 6f88a7394..e2b3083ee 100644 --- a/lib/cache/layeredcache_test.go +++ b/lib/cache/layeredcache_test.go @@ -1,7 +1,6 @@ package ccache import ( - "os" "strconv" "sync/atomic" "testing" @@ -225,19 +224,12 @@ func Test_LayeredCache_RemovesOldestItemWhenFull(t *testing.T) { } func Test_LayeredCache_ResizeOnTheFly(t *testing.T) { - // On a busy system or during a slow run, the cleanup might take longer. - // When this happens, the test continues - // and runs its assertions (e.g., assert.Equal(t, cache.GetDropped(), 2)) - // before the cache has actually been pruned, causing the test to fail. - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } cache := Layered(Configure().MaxSize(9).ItemsToPrune(1)) for i := 0; i < 5; i++ { cache.Set(strconv.Itoa(i), "a", i, time.Minute) } cache.SetMaxSize(3) - time.Sleep(time.Millisecond * 10) + time.Sleep(time.Millisecond * 100) assert.Equal(t, cache.GetDropped(), 2) assert.Nil(t, cache.Get("0", "a")) assert.Nil(t, cache.Get("1", "a")) @@ -246,7 +238,7 @@ func Test_LayeredCache_ResizeOnTheFly(t *testing.T) { assert.Equal(t, cache.Get("4", "a").Value(), 4) cache.Set("5", "a", 5, time.Minute) - time.Sleep(time.Millisecond * 5) + time.Sleep(time.Millisecond * 100) assert.Equal(t, cache.GetDropped(), 1) assert.Nil(t, cache.Get("2", "a")) assert.Equal(t, cache.Get("3", "a").Value(), 3) @@ -255,7 +247,7 @@ func Test_LayeredCache_ResizeOnTheFly(t *testing.T) { cache.SetMaxSize(10) cache.Set("6", "a", 6, time.Minute) - time.Sleep(time.Millisecond * 10) + time.Sleep(time.Millisecond * 100) assert.Equal(t, cache.GetDropped(), 0) assert.Equal(t, cache.Get("3", "a").Value(), 3) assert.Equal(t, cache.Get("4", "a").Value(), 4) diff --git a/lib/fasthttp/allocation_test.go b/lib/fasthttp/allocation_test.go deleted file mode 100644 index 55a0e958f..000000000 --- a/lib/fasthttp/allocation_test.go +++ /dev/null @@ -1,94 +0,0 @@ -//go:build !race -// +build !race - -package fasthttp - -import ( - "net" - "os" - - "testing" -) - -func TestAllocationServeConn(t *testing.T) { - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - } - - rw := &readWriter{} - // Make space for the request and response here so it - // doesn't allocate within the test. - rw.r.Grow(1024) - rw.w.Grow(1024) - - n := testing.AllocsPerRun(100, func() { - rw.r.WriteString("GET / HTTP/1.1\r\nHost: google.com\r\nCookie: foo=bar\r\n\r\n") - if err := s.ServeConn(rw); err != nil { - t.Fatal(err) - } - - // Reset the write buffer to make space for the next response. - rw.w.Reset() - }) - - if n != 0 { - t.Fatalf("expected 0 allocations, got %f", n) - } -} - -func TestAllocationClient(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - - ln, err := net.Listen("tcp4", "127.0.0.1:0") - if err != nil { - t.Fatalf("cannot listen: %v", err) - } - defer ln.Close() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - } - go s.Serve(ln) //nolint:errcheck - - c := &Client{} - url := "http://test:test@" + ln.Addr().String() + "/foo?bar=baz" - - n := testing.AllocsPerRun(100, func() { - req := defaultHTTPPool.AcquireRequest() - res := defaultHTTPPool.AcquireResponse() - - req.SetRequestURI(url) - if err := c.Do(req, res); err != nil { - t.Fatal(err) - } - - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(res) - }) - - if n != 0 { - t.Fatalf("expected 0 allocations, got %f", n) - } -} - -func TestAllocationURI(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - - uri := []byte("http://username:password@hello.%e4%b8%96%e7%95%8c.com/some/path?foo=bar#test") - - n := testing.AllocsPerRun(100, func() { - u := AcquireURI() - u.Parse(nil, uri) //nolint:errcheck - ReleaseURI(u) - }) - - if n != 0 { - t.Fatalf("expected 0 allocations, got %f", n) - } -} diff --git a/lib/fasthttp/args_test.go b/lib/fasthttp/args_test.go deleted file mode 100644 index a518a1186..000000000 --- a/lib/fasthttp/args_test.go +++ /dev/null @@ -1,623 +0,0 @@ -package fasthttp - -import ( - "bytes" - "fmt" - "net/url" - "reflect" - "strings" - "testing" - "time" - - "infini.sh/framework/lib/bytebufferpool" -) - -func TestDecodeArgAppend(t *testing.T) { - t.Parallel() - - testDecodeArgAppend(t, "", "") - testDecodeArgAppend(t, "foobar", "foobar") - testDecodeArgAppend(t, "тест", "тест") - testDecodeArgAppend(t, "a%", "a%") - testDecodeArgAppend(t, "%a%21", "%a!") - testDecodeArgAppend(t, "ab%test", "ab%test") - testDecodeArgAppend(t, "d%тестF", "d%тестF") - testDecodeArgAppend(t, "a%\xffb%20c", "a%\xffb c") - testDecodeArgAppend(t, "foo%20bar", "foo bar") - testDecodeArgAppend(t, "f.o%2C1%3A2%2F4=%7E%60%21%40%23%24%25%5E%26*%28%29_-%3D%2B%5C%7C%2F%5B%5D%7B%7D%3B%3A%27%22%3C%3E%2C.%2F%3F", - "f.o,1:2/4=~`!@#$%^&*()_-=+\\|/[]{};:'\"<>,./?") -} - -func testDecodeArgAppend(t *testing.T, s, expectedResult string) { - result := decodeArgAppend(nil, []byte(s)) - if string(result) != expectedResult { - t.Fatalf("unexpected decodeArgAppend(%q)=%q; expecting %q", s, result, expectedResult) - } -} - -func TestArgsAdd(t *testing.T) { - t.Parallel() - - var a Args - a.Add("foo", "bar") - a.Add("foo", "baz") - a.Add("foo", "1") - a.Add("ba", "23") - a.Add("foo", "") - a.AddNoValue("foo") - if a.Len() != 6 { - t.Fatalf("unexpected number of elements: %d. Expecting 6", a.Len()) - } - s := a.String() - expectedS := "foo=bar&foo=baz&foo=1&ba=23&foo=&foo" - if s != expectedS { - t.Fatalf("unexpected result: %q. Expecting %q", s, expectedS) - } - - a.Sort(bytes.Compare) - ss := a.String() - expectedSS := "ba=23&foo=&foo&foo=1&foo=bar&foo=baz" - if ss != expectedSS { - t.Fatalf("unexpected result: %q. Expecting %q", ss, expectedSS) - } - - var a1 Args - a1.Parse(s) - if a1.Len() != 6 { - t.Fatalf("unexpected number of elements: %d. Expecting 6", a.Len()) - } - - var barFound, bazFound, oneFound, emptyFound1, emptyFound2, baFound bool - a1.VisitAll(func(k, v []byte) { - switch string(k) { - case "foo": - switch string(v) { - case "bar": - barFound = true - case "baz": - bazFound = true - case "1": - oneFound = true - case "": - if emptyFound1 { - emptyFound2 = true - } else { - emptyFound1 = true - } - default: - t.Fatalf("unexpected value %q", v) - } - case "ba": - if string(v) != "23" { - t.Fatalf("unexpected value: %q. Expecting %q", v, "23") - } - baFound = true - default: - t.Fatalf("unexpected key found %q", k) - } - }) - if !barFound || !bazFound || !oneFound || !emptyFound1 || !emptyFound2 || !baFound { - t.Fatalf("something is missing: %v, %v, %v, %v, %v, %v", barFound, bazFound, oneFound, emptyFound1, emptyFound2, baFound) - } -} - -func TestArgsAcquireReleaseSequential(t *testing.T) { - testArgsAcquireRelease(t) -} - -func TestArgsAcquireReleaseConcurrent(t *testing.T) { - ch := make(chan struct{}, 10) - for i := 0; i < 10; i++ { - go func() { - testArgsAcquireRelease(t) - ch <- struct{}{} - }() - } - for i := 0; i < 10; i++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } -} - -func testArgsAcquireRelease(t *testing.T) { - a := AcquireArgs() - - for i := 0; i < 10; i++ { - k := fmt.Sprintf("key_%d", i) - v := fmt.Sprintf("value_%d", i*3+123) - a.Set(k, v) - } - - s := a.String() - a.Reset() - a.Parse(s) - - for i := 0; i < 10; i++ { - k := fmt.Sprintf("key_%d", i) - expectedV := fmt.Sprintf("value_%d", i*3+123) - v := a.Peek(k) - if string(v) != expectedV { - t.Fatalf("unexpected value %q for key %q. Expecting %q", v, k, expectedV) - } - } - - ReleaseArgs(a) -} - -func TestArgsPeekMulti(t *testing.T) { - t.Parallel() - - var a Args - a.Parse("foo=123&bar=121&foo=321&foo=&barz=sdf") - - vv := a.PeekMulti("foo") - expectedVV := [][]byte{ - []byte("123"), - []byte("321"), - []byte(nil), - } - if !reflect.DeepEqual(vv, expectedVV) { - t.Fatalf("unexpected vv\n%#v\nExpecting\n%#v\n", vv, expectedVV) - } - - vv = a.PeekMulti("aaaa") - if len(vv) > 0 { - t.Fatalf("expecting empty result for non-existing key. Got %#v", vv) - } - - vv = a.PeekMulti("bar") - expectedVV = [][]byte{[]byte("121")} - if !reflect.DeepEqual(vv, expectedVV) { - t.Fatalf("unexpected vv\n%#v\nExpecting\n%#v\n", vv, expectedVV) - } -} - -func TestArgsEscape(t *testing.T) { - t.Parallel() - - testArgsEscape(t, "foo", "bar", "foo=bar") - - // Test all characters - k := "f.o,1:2/4" - var v = make([]byte, 256) - for i := 0; i < 256; i++ { - v[i] = byte(i) - } - u := url.Values{} - u.Add(k, string(v)) - testArgsEscape(t, k, string(v), u.Encode()) -} - -func testArgsEscape(t *testing.T, k, v, expectedS string) { - var a Args - a.Set(k, v) - s := a.String() - if s != expectedS { - t.Fatalf("unexpected args %q. Expecting %q. k=%q, v=%q", s, expectedS, k, v) - } -} - -func TestPathEscape(t *testing.T) { - t.Parallel() - - testPathEscape(t, "/foo/bar") - testPathEscape(t, "") - testPathEscape(t, "/") - testPathEscape(t, "//") - testPathEscape(t, "*") // See https://github.com/golang/go/issues/11202 - - // Test all characters - var pathSegment = make([]byte, 256) - for i := 0; i < 256; i++ { - pathSegment[i] = byte(i) - } - testPathEscape(t, "/foo/"+string(pathSegment)) -} - -func testPathEscape(t *testing.T, s string) { - u := url.URL{Path: s} - expectedS := u.EscapedPath() - res := string(appendQuotedPath(nil, []byte(s))) - if res != expectedS { - t.Fatalf("unexpected args %q. Expecting %q.", res, expectedS) - } -} - -func TestArgsWriteTo(t *testing.T) { - t.Parallel() - - s := "foo=bar&baz=123&aaa=bbb" - - var a Args - a.Parse(s) - - var w bytebufferpool.ByteBuffer - n, err := a.WriteTo(&w) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if n != int64(len(s)) { - t.Fatalf("unexpected n: %d. Expecting %d", n, len(s)) - } - result := string(w.B) - if result != s { - t.Fatalf("unexpected result %q. Expecting %q", result, s) - } -} - -func TestArgsGetBool(t *testing.T) { - t.Parallel() - - testArgsGetBool(t, "", false) - testArgsGetBool(t, "0", false) - testArgsGetBool(t, "n", false) - testArgsGetBool(t, "no", false) - testArgsGetBool(t, "1", true) - testArgsGetBool(t, "y", true) - testArgsGetBool(t, "yes", true) - - testArgsGetBool(t, "123", false) - testArgsGetBool(t, "foobar", false) -} - -func testArgsGetBool(t *testing.T, value string, expectedResult bool) { - var a Args - a.Parse("v=" + value) - - result := a.GetBool("v") - if result != expectedResult { - t.Fatalf("unexpected result %v. Expecting %v for value %q", result, expectedResult, value) - } -} - -func TestArgsUint(t *testing.T) { - t.Parallel() - - var a Args - a.SetUint("foo", 123) - a.SetUint("bar", 0) - a.SetUint("aaaa", 34566) - - expectedS := "foo=123&bar=0&aaaa=34566" - s := string(a.QueryString()) - if s != expectedS { - t.Fatalf("unexpected args %q. Expecting %q", s, expectedS) - } - - if a.GetUintOrZero("foo") != 123 { - t.Fatalf("unexpected arg value %d. Expecting %d", a.GetUintOrZero("foo"), 123) - } - if a.GetUintOrZero("bar") != 0 { - t.Fatalf("unexpected arg value %d. Expecting %d", a.GetUintOrZero("bar"), 0) - } - if a.GetUintOrZero("aaaa") != 34566 { - t.Fatalf("unexpected arg value %d. Expecting %d", a.GetUintOrZero("aaaa"), 34566) - } - - if string(a.Peek("foo")) != "123" { - t.Fatalf("unexpected arg value %q. Expecting %q", a.Peek("foo"), "123") - } - if string(a.Peek("bar")) != "0" { - t.Fatalf("unexpected arg value %q. Expecting %q", a.Peek("bar"), "0") - } - if string(a.Peek("aaaa")) != "34566" { - t.Fatalf("unexpected arg value %q. Expecting %q", a.Peek("aaaa"), "34566") - } -} - -func TestArgsCopyTo(t *testing.T) { - t.Parallel() - - var a Args - - // empty args - testCopyTo(t, &a) - - a.Set("foo", "bar") - testCopyTo(t, &a) - - a.Set("xxx", "yyy") - a.AddNoValue("ba") - testCopyTo(t, &a) - - a.Del("foo") - testCopyTo(t, &a) -} - -func testCopyTo(t *testing.T, a *Args) { - keys := make(map[string]struct{}) - a.VisitAll(func(k, _ []byte) { - keys[string(k)] = struct{}{} - }) - - var b Args - a.CopyTo(&b) - - if !reflect.DeepEqual(*a, b) { //nolint - t.Fatalf("ArgsCopyTo fail, a: \n%+v\nb: \n%+v\n", *a, b) //nolint - } - - b.VisitAll(func(k, _ []byte) { - if _, ok := keys[string(k)]; !ok { - t.Fatalf("unexpected key %q after copying from %q", k, a.String()) - } - delete(keys, string(k)) - }) - if len(keys) > 0 { - t.Fatalf("missing keys %#v after copying from %q", keys, a.String()) - } -} - -func TestArgsVisitAll(t *testing.T) { - t.Parallel() - - var a Args - a.Set("foo", "bar") - - i := 0 - a.VisitAll(func(k, v []byte) { - if string(k) != "foo" { - t.Fatalf("unexpected key %q. Expected %q", k, "foo") - } - if string(v) != "bar" { - t.Fatalf("unexpected value %q. Expected %q", v, "bar") - } - i++ - }) - if i != 1 { - t.Fatalf("unexpected number of VisitAll calls: %d. Expected %d", i, 1) - } -} - -func TestArgsStringCompose(t *testing.T) { - t.Parallel() - - var a Args - a.Set("foo", "bar") - a.Set("aa", "bbb") - a.Set("привет", "мир") - a.SetNoValue("bb") - a.Set("", "xxxx") - a.Set("cvx", "") - a.SetNoValue("novalue") - - expectedS := "foo=bar&aa=bbb&%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82=%D0%BC%D0%B8%D1%80&bb&=xxxx&cvx=&novalue" - s := a.String() - if s != expectedS { - t.Fatalf("Unexpected string %q. Exected %q", s, expectedS) - } -} - -func TestArgsString(t *testing.T) { - t.Parallel() - - var a Args - - testArgsString(t, &a, "") - testArgsString(t, &a, "foobar") - testArgsString(t, &a, "foo=bar") - testArgsString(t, &a, "foo=bar&baz=sss") - testArgsString(t, &a, "") - testArgsString(t, &a, "f+o=x.x%2A-_8x%D0%BF%D1%80%D0%B8%D0%B2%D0%B5aaa&sdf=ss") - testArgsString(t, &a, "=asdfsdf") -} - -func testArgsString(t *testing.T, a *Args, s string) { - a.Parse(s) - s1 := a.String() - if s != s1 { - t.Fatalf("Unexpected args %q. Expected %q", s1, s) - } -} - -func TestArgsSetGetDel(t *testing.T) { - t.Parallel() - - var a Args - - if len(a.Peek("foo")) > 0 { - t.Fatalf("Unexpected value: %q", a.Peek("foo")) - } - if len(a.Peek("")) > 0 { - t.Fatalf("Unexpected value: %q", a.Peek("")) - } - a.Del("xxx") - - for j := 0; j < 3; j++ { - for i := 0; i < 10; i++ { - k := fmt.Sprintf("foo%d", i) - v := fmt.Sprintf("bar_%d", i) - a.Set(k, v) - if string(a.Peek(k)) != v { - t.Fatalf("Unexpected value: %q. Expected %q", a.Peek(k), v) - } - } - } - for i := 0; i < 10; i++ { - k := fmt.Sprintf("foo%d", i) - v := fmt.Sprintf("bar_%d", i) - if string(a.Peek(k)) != v { - t.Fatalf("Unexpected value: %q. Expected %q", a.Peek(k), v) - } - a.Del(k) - if string(a.Peek(k)) != "" { - t.Fatalf("Unexpected value: %q. Expected %q", a.Peek(k), "") - } - } - - a.Parse("aaa=xxx&bb=aa") - if string(a.Peek("foo0")) != "" { - t.Fatalf("Unexpected value %q", a.Peek("foo0")) - } - if string(a.Peek("aaa")) != "xxx" { - t.Fatalf("Unexpected value %q. Expected %q", a.Peek("aaa"), "xxx") - } - if string(a.Peek("bb")) != "aa" { - t.Fatalf("Unexpected value %q. Expected %q", a.Peek("bb"), "aa") - } - - for i := 0; i < 10; i++ { - k := fmt.Sprintf("xx%d", i) - v := fmt.Sprintf("yy%d", i) - a.Set(k, v) - if string(a.Peek(k)) != v { - t.Fatalf("Unexpected value: %q. Expected %q", a.Peek(k), v) - } - } - for i := 5; i < 10; i++ { - k := fmt.Sprintf("xx%d", i) - v := fmt.Sprintf("yy%d", i) - if string(a.Peek(k)) != v { - t.Fatalf("Unexpected value: %q. Expected %q", a.Peek(k), v) - } - a.Del(k) - if string(a.Peek(k)) != "" { - t.Fatalf("Unexpected value: %q. Expected %q", a.Peek(k), "") - } - } -} - -func TestArgsParse(t *testing.T) { - t.Parallel() - - var a Args - - // empty args - testArgsParse(t, &a, "", 0, "foo=", "bar=", "=") - - // arg without value - testArgsParse(t, &a, "foo1", 1, "foo=", "bar=", "=") - - // arg without value, but with equal sign - testArgsParse(t, &a, "foo2=", 1, "foo=", "bar=", "=") - - // arg with value - testArgsParse(t, &a, "foo3=bar1", 1, "foo3=bar1", "bar=", "=") - - // empty key - testArgsParse(t, &a, "=bar2", 1, "foo=", "=bar2", "bar2=") - - // missing kv - testArgsParse(t, &a, "&&&&", 0, "foo=", "bar=", "=") - - // multiple values with the same key - testArgsParse(t, &a, "x=1&x=2&x=3", 3, "x=1") - - // multiple args - testArgsParse(t, &a, "&&&qw=er&tyx=124&&&zxc_ss=2234&&", 3, "qw=er", "tyx=124", "zxc_ss=2234") - - // multiple args without values - testArgsParse(t, &a, "&&a&&b&&bar&baz", 4, "a=", "b=", "bar=", "baz=") - - // values with '=' - testArgsParse(t, &a, "zz=1&k=v=v=a=a=s", 2, "k=v=v=a=a=s", "zz=1") - - // mixed '=' and '&' - testArgsParse(t, &a, "sss&z=dsf=&df", 3, "sss=", "z=dsf=", "df=") - - // encoded args - testArgsParse(t, &a, "f+o%20o=%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82+test", 1, "f o o=привет test") - - // invalid percent encoding - testArgsParse(t, &a, "f%=x&qw%z=d%0k%20p&%%20=%%%20x", 3, "f%=x", "qw%z=d%0k p", "% =%% x") - - // special chars - testArgsParse(t, &a, "a.b,c:d/e=f.g,h:i/q", 1, "a.b,c:d/e=f.g,h:i/q") -} - -func TestArgsHas(t *testing.T) { - t.Parallel() - - var a Args - - // single arg - testArgsHas(t, &a, "foo", "foo") - testArgsHasNot(t, &a, "foo", "bar", "baz", "") - - // multi args without values - testArgsHas(t, &a, "foo&bar", "foo", "bar") - testArgsHasNot(t, &a, "foo&bar", "", "aaaa") - - // multi args - testArgsHas(t, &a, "b=xx&=aaa&c=", "b", "", "c") - testArgsHasNot(t, &a, "b=xx&=aaa&c=", "xx", "aaa", "foo") - - // encoded args - testArgsHas(t, &a, "a+b=c+d%20%20e", "a b") - testArgsHasNot(t, &a, "a+b=c+d", "a+b", "c+d") -} - -func testArgsHas(t *testing.T, a *Args, s string, expectedKeys ...string) { - a.Parse(s) - for _, key := range expectedKeys { - if !a.Has(key) { - t.Fatalf("Missing key %q in %q", key, s) - } - } -} - -func testArgsHasNot(t *testing.T, a *Args, s string, unexpectedKeys ...string) { - a.Parse(s) - for _, key := range unexpectedKeys { - if a.Has(key) { - t.Fatalf("Unexpected key %q in %q", key, s) - } - } -} - -func testArgsParse(t *testing.T, a *Args, s string, expectedLen int, expectedArgs ...string) { - a.Parse(s) - if a.Len() != expectedLen { - t.Fatalf("Unexpected args len %d. Expected %d. s=%q", a.Len(), expectedLen, s) - } - for _, xx := range expectedArgs { - tmp := strings.SplitN(xx, "=", 2) - k := tmp[0] - v := tmp[1] - buf := a.Peek(k) - if string(buf) != v { - t.Fatalf("Unexpected value for key=%q: %q. Expected %q. s=%q", k, buf, v, s) - } - } -} - -func TestArgsDeleteAll(t *testing.T) { - t.Parallel() - var a Args - a.Add("q1", "foo") - a.Add("q1", "bar") - a.Add("q1", "baz") - a.Add("q1", "quux") - a.Add("q2", "1234") - a.Del("q1") - if a.Len() != 1 || a.Has("q1") { - t.Fatalf("Expected q1 arg to be completely deleted. Current Args: %q", a.String()) - } -} - -func TestIssue932(t *testing.T) { - t.Parallel() - var a []argsKV - - a = setArg(a, "t1", "ok", argsHasValue) - a = setArg(a, "t2", "", argsHasValue) - a = setArg(a, "t1", "", argsHasValue) - a = setArgBytes(a, s2b("t3"), []byte{}, argsHasValue) - a = setArgBytes(a, s2b("t4"), nil, argsHasValue) - - if peekArgStr(a, "t1") == nil { - t.Error("nil not expected for t1") - } - if peekArgStr(a, "t2") == nil { - t.Error("nil not expected for t2") - } - if peekArgStr(a, "t3") == nil { - t.Error("nil not expected for t3") - } - if peekArgStr(a, "t4") != nil { - t.Error("nil expected for t4") - } -} diff --git a/lib/fasthttp/args_timing_test.go b/lib/fasthttp/args_timing_test.go deleted file mode 100644 index d6e9e985d..000000000 --- a/lib/fasthttp/args_timing_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package fasthttp - -import ( - "bytes" - "testing" -) - -func BenchmarkArgsParse(b *testing.B) { - s := []byte("foo=bar&baz=qqq&aaaaa=bbbb") - b.RunParallel(func(pb *testing.PB) { - var a Args - for pb.Next() { - a.ParseBytes(s) - } - }) -} - -func BenchmarkArgsPeek(b *testing.B) { - value := []byte("foobarbaz1234") - key := "foobarbaz" - b.RunParallel(func(pb *testing.PB) { - var a Args - a.SetBytesV(key, value) - for pb.Next() { - if !bytes.Equal(a.Peek(key), value) { - b.Fatalf("unexpected arg value %q. Expecting %q", a.Peek(key), value) - } - } - }) -} diff --git a/lib/fasthttp/brotli_test.go b/lib/fasthttp/brotli_test.go deleted file mode 100644 index 2070fb599..000000000 --- a/lib/fasthttp/brotli_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package fasthttp - -import ( - "bytes" - "fmt" - "io/ioutil" - "testing" -) - -func TestBrotliBytesSerial(t *testing.T) { - t.Parallel() - - if err := testBrotliBytes(); err != nil { - t.Fatal(err) - } -} - -func TestBrotliBytesConcurrent(t *testing.T) { - t.Parallel() - - if err := testConcurrent(10, testBrotliBytes); err != nil { - t.Fatal(err) - } -} - -func testBrotliBytes() error { - for _, s := range compressTestcases { - if err := testBrotliBytesSingleCase(s); err != nil { - return err - } - } - return nil -} - -func testBrotliBytesSingleCase(s string) error { - prefix := []byte("foobar") - brotlipedS := AppendBrotliBytes(prefix, []byte(s)) - if !bytes.Equal(brotlipedS[:len(prefix)], prefix) { - return fmt.Errorf("unexpected prefix when compressing %q: %q. Expecting %q", s, brotlipedS[:len(prefix)], prefix) - } - - unbrotliedS, err := AppendUnbrotliBytes(prefix, brotlipedS[len(prefix):]) - if err != nil { - return fmt.Errorf("unexpected error when uncompressing %q: %w", s, err) - } - if !bytes.Equal(unbrotliedS[:len(prefix)], prefix) { - return fmt.Errorf("unexpected prefix when uncompressing %q: %q. Expecting %q", s, unbrotliedS[:len(prefix)], prefix) - } - unbrotliedS = unbrotliedS[len(prefix):] - if string(unbrotliedS) != s { - return fmt.Errorf("unexpected uncompressed string %q. Expecting %q", unbrotliedS, s) - } - return nil -} - -func TestBrotliCompressSerial(t *testing.T) { - t.Parallel() - - if err := testBrotliCompress(); err != nil { - t.Fatal(err) - } -} - -func TestBrotliCompressConcurrent(t *testing.T) { - t.Parallel() - - if err := testConcurrent(10, testBrotliCompress); err != nil { - t.Fatal(err) - } -} - -func testBrotliCompress() error { - for _, s := range compressTestcases { - if err := testBrotliCompressSingleCase(s); err != nil { - return err - } - } - return nil -} - -func testBrotliCompressSingleCase(s string) error { - var buf bytes.Buffer - zw := acquireStacklessBrotliWriter(&buf, CompressDefaultCompression) - if _, err := zw.Write([]byte(s)); err != nil { - return fmt.Errorf("unexpected error: %w. s=%q", err, s) - } - releaseStacklessBrotliWriter(zw, CompressDefaultCompression) - - zr, err := acquireBrotliReader(&buf) - if err != nil { - return fmt.Errorf("unexpected error: %w. s=%q", err, s) - } - body, err := ioutil.ReadAll(zr) - if err != nil { - return fmt.Errorf("unexpected error: %w. s=%q", err, s) - } - if string(body) != s { - return fmt.Errorf("unexpected string after decompression: %q. Expecting %q", body, s) - } - releaseBrotliReader(zr) - return nil -} diff --git a/lib/fasthttp/bytesconv_32_test.go b/lib/fasthttp/bytesconv_32_test.go deleted file mode 100644 index 3f5d5ded8..000000000 --- a/lib/fasthttp/bytesconv_32_test.go +++ /dev/null @@ -1,60 +0,0 @@ -//go:build !amd64 && !arm64 && !ppc64 && !ppc64le && !s390x -// +build !amd64,!arm64,!ppc64,!ppc64le,!s390x - -package fasthttp - -import ( - "testing" -) - -func TestWriteHexInt(t *testing.T) { - t.Parallel() - - testWriteHexInt(t, 0, "0") - testWriteHexInt(t, 1, "1") - testWriteHexInt(t, 0x123, "123") - testWriteHexInt(t, 0x7fffffff, "7fffffff") -} - -func TestAppendUint(t *testing.T) { - t.Parallel() - - testAppendUint(t, 0) - testAppendUint(t, 123) - testAppendUint(t, 0x7fffffff) - - for i := 0; i < 2345; i++ { - testAppendUint(t, i) - } -} - -func TestReadHexIntSuccess(t *testing.T) { - t.Parallel() - - testReadHexIntSuccess(t, "0", 0) - testReadHexIntSuccess(t, "fF", 0xff) - testReadHexIntSuccess(t, "00abc", 0xabc) - testReadHexIntSuccess(t, "7ffffff", 0x7ffffff) - testReadHexIntSuccess(t, "000", 0) - testReadHexIntSuccess(t, "1234ZZZ", 0x1234) -} - -func TestParseUintError32(t *testing.T) { - t.Parallel() - - // Overflow by last digit: 2 ** 32 / 2 * 10 ** n - testParseUintError(t, "2147483648") - testParseUintError(t, "21474836480") - testParseUintError(t, "214748364800") -} - -func TestParseUintSuccess(t *testing.T) { - t.Parallel() - - testParseUintSuccess(t, "0", 0) - testParseUintSuccess(t, "123", 123) - testParseUintSuccess(t, "123456789", 123456789) - - // Max supported value: 2 ** 32 / 2 - 1 - testParseUintSuccess(t, "2147483647", 2147483647) -} diff --git a/lib/fasthttp/bytesconv_64_test.go b/lib/fasthttp/bytesconv_64_test.go deleted file mode 100644 index 06898090a..000000000 --- a/lib/fasthttp/bytesconv_64_test.go +++ /dev/null @@ -1,62 +0,0 @@ -//go:build amd64 || arm64 || ppc64 || ppc64le || s390x -// +build amd64 arm64 ppc64 ppc64le s390x - -package fasthttp - -import ( - "testing" -) - -func TestWriteHexInt(t *testing.T) { - t.Parallel() - - testWriteHexInt(t, 0, "0") - testWriteHexInt(t, 1, "1") - testWriteHexInt(t, 0x123, "123") - testWriteHexInt(t, 0x7fffffffffffffff, "7fffffffffffffff") -} - -func TestAppendUint(t *testing.T) { - t.Parallel() - - testAppendUint(t, 0) - testAppendUint(t, 123) - testAppendUint(t, 0x7fffffffffffffff) - - for i := 0; i < 2345; i++ { - testAppendUint(t, i) - } -} - -func TestReadHexIntSuccess(t *testing.T) { - t.Parallel() - - testReadHexIntSuccess(t, "0", 0) - testReadHexIntSuccess(t, "fF", 0xff) - testReadHexIntSuccess(t, "00abc", 0xabc) - testReadHexIntSuccess(t, "7fffffff", 0x7fffffff) - testReadHexIntSuccess(t, "000", 0) - testReadHexIntSuccess(t, "1234ZZZ", 0x1234) - testReadHexIntSuccess(t, "7ffffffffffffff", 0x7ffffffffffffff) -} - -func TestParseUintError64(t *testing.T) { - t.Parallel() - - // Overflow by last digit: 2 ** 64 / 2 * 10 ** n - testParseUintError(t, "9223372036854775808") - testParseUintError(t, "92233720368547758080") - testParseUintError(t, "922337203685477580800") -} - -func TestParseUintSuccess(t *testing.T) { - t.Parallel() - - testParseUintSuccess(t, "0", 0) - testParseUintSuccess(t, "123", 123) - testParseUintSuccess(t, "1234567890", 1234567890) - testParseUintSuccess(t, "123456789012345678", 123456789012345678) - - // Max supported value: 2 ** 64 / 2 - 1 - testParseUintSuccess(t, "9223372036854775807", 9223372036854775807) -} diff --git a/lib/fasthttp/bytesconv_test.go b/lib/fasthttp/bytesconv_test.go deleted file mode 100644 index 6ebb78311..000000000 --- a/lib/fasthttp/bytesconv_test.go +++ /dev/null @@ -1,349 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "fmt" - "html" - "net" - "net/url" - "testing" - "time" - - "infini.sh/framework/lib/bytebufferpool" -) - -func TestAppendQuotedArg(t *testing.T) { - t.Parallel() - - // Sync with url.QueryEscape - allcases := make([]byte, 256) - for i := 0; i < 256; i++ { - allcases[i] = byte(i) - } - res := string(AppendQuotedArg(nil, allcases)) - expect := url.QueryEscape(string(allcases)) - if res != expect { - t.Fatalf("unexpected string %q. Expecting %q.", res, expect) - } -} - -func TestAppendHTMLEscape(t *testing.T) { - t.Parallel() - - // Sync with html.EscapeString - allcases := make([]byte, 256) - for i := 0; i < 256; i++ { - allcases[i] = byte(i) - } - res := string(AppendHTMLEscape(nil, string(allcases))) - expect := string(html.EscapeString(string(allcases))) - if res != expect { - t.Fatalf("unexpected string %q. Expecting %q.", res, expect) - } - - testAppendHTMLEscape(t, "", "") - testAppendHTMLEscape(t, "<", "<") - testAppendHTMLEscape(t, "a", "a") - testAppendHTMLEscape(t, `><"''`, "><"''") - testAppendHTMLEscape(t, "foaxxx", "fo<b x='ss'>a</b>xxx") -} - -func testAppendHTMLEscape(t *testing.T, s, expectedS string) { - buf := AppendHTMLEscapeBytes(nil, []byte(s)) - if string(buf) != expectedS { - t.Fatalf("unexpected html-escaped string %q. Expecting %q. Original string %q", buf, expectedS, s) - } -} - -func TestParseIPv4(t *testing.T) { - t.Parallel() - - testParseIPv4(t, "0.0.0.0", true) - testParseIPv4(t, "255.255.255.255", true) - testParseIPv4(t, "123.45.67.89", true) - - // ipv6 shouldn't work - testParseIPv4(t, "2001:4860:0:2001::68", false) - - // invalid ip - testParseIPv4(t, "foobar", false) - testParseIPv4(t, "1.2.3", false) - testParseIPv4(t, "123.456.789.11", false) -} - -func testParseIPv4(t *testing.T, ipStr string, isValid bool) { - ip, err := ParseIPv4(nil, []byte(ipStr)) - if isValid { - if err != nil { - t.Fatalf("unexpected error when parsing ip %q: %v", ipStr, err) - } - s := string(AppendIPv4(nil, ip)) - if s != ipStr { - t.Fatalf("unexpected ip parsed %q. Expecting %q", s, ipStr) - } - } else { - if err == nil { - t.Fatalf("expecting error when parsing ip %q", ipStr) - } - } -} - -func TestAppendIPv4(t *testing.T) { - t.Parallel() - - testAppendIPv4(t, "0.0.0.0", true) - testAppendIPv4(t, "127.0.0.1", true) - testAppendIPv4(t, "8.8.8.8", true) - testAppendIPv4(t, "123.45.67.89", true) - - // ipv6 shouldn't work - testAppendIPv4(t, "2001:4860:0:2001::68", false) -} - -func testAppendIPv4(t *testing.T, ipStr string, isValid bool) { - ip := net.ParseIP(ipStr) - if ip == nil { - t.Fatalf("cannot parse ip %q", ipStr) - } - s := string(AppendIPv4(nil, ip)) - if isValid { - if s != ipStr { - t.Fatalf("unexpected ip %q. Expecting %q", s, ipStr) - } - } else { - ipStr = "non-v4 ip passed to AppendIPv4" - if s != ipStr { - t.Fatalf("unexpected ip %q. Expecting %q", s, ipStr) - } - } -} - -func testAppendUint(t *testing.T, n int) { - expectedS := fmt.Sprintf("%d", n) - s := AppendUint(nil, n) - if string(s) != expectedS { - t.Fatalf("unexpected uint %q. Expecting %q. n=%d", s, expectedS, n) - } -} - -func testWriteHexInt(t *testing.T, n int, expectedS string) { - var w bytebufferpool.ByteBuffer - bw := bufio.NewWriter(&w) - if err := writeHexInt(bw, n); err != nil { - t.Fatalf("unexpected error when writing hex %x: %v", n, err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error when flushing hex %x: %v", n, err) - } - s := string(w.B) - if s != expectedS { - t.Fatalf("unexpected hex after writing %q. Expected %q", s, expectedS) - } -} - -func TestReadHexIntError(t *testing.T) { - t.Parallel() - - testReadHexIntError(t, "") - testReadHexIntError(t, "ZZZ") - testReadHexIntError(t, "-123") - testReadHexIntError(t, "+434") -} - -func testReadHexIntError(t *testing.T, s string) { - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - n, err := readHexInt(br) - if err == nil { - t.Fatalf("expecting error when reading hex int %q", s) - } - if n >= 0 { - t.Fatalf("unexpected hex value read %d for hex int %q. must be negative", n, s) - } -} - -func testReadHexIntSuccess(t *testing.T, s string, expectedN int) { - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - n, err := readHexInt(br) - if err != nil { - t.Fatalf("unexpected error: %v. s=%q", err, s) - } - if n != expectedN { - t.Fatalf("unexpected hex int %d. Expected %d. s=%q", n, expectedN, s) - } -} - -func TestAppendHTTPDate(t *testing.T) { - t.Parallel() - - d := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - s := string(AppendHTTPDate(nil, d)) - expectedS := "Tue, 10 Nov 2009 23:00:00 GMT" - if s != expectedS { - t.Fatalf("unexpected date %q. Expecting %q", s, expectedS) - } - - b := []byte("prefix") - s = string(AppendHTTPDate(b, d)) - if s[:len(b)] != string(b) { - t.Fatalf("unexpected prefix %q. Expecting %q", s[:len(b)], b) - } - s = s[len(b):] - if s != expectedS { - t.Fatalf("unexpected date %q. Expecting %q", s, expectedS) - } -} - -func TestParseUintError(t *testing.T) { - t.Parallel() - - // empty string - testParseUintError(t, "") - - // negative value - testParseUintError(t, "-123") - - // non-num - testParseUintError(t, "foobar234") - - // non-num chars at the end - testParseUintError(t, "123w") - - // floating point num - testParseUintError(t, "1234.545") - - // too big num - testParseUintError(t, "12345678901234567890") - testParseUintError(t, "1234567890123456789012") -} - -func TestParseUfloatSuccess(t *testing.T) { - t.Parallel() - - testParseUfloatSuccess(t, "0", 0) - testParseUfloatSuccess(t, "1.", 1.) - testParseUfloatSuccess(t, ".1", 0.1) - testParseUfloatSuccess(t, "123.456", 123.456) - testParseUfloatSuccess(t, "123", 123) - testParseUfloatSuccess(t, "1234e2", 1234e2) - testParseUfloatSuccess(t, "1234E-5", 1234e-5) - testParseUfloatSuccess(t, "1.234e+3", 1.234e+3) -} - -func TestParseUfloatError(t *testing.T) { - t.Parallel() - - // empty num - testParseUfloatError(t, "") - - // negative num - testParseUfloatError(t, "-123.53") - - // non-num chars - testParseUfloatError(t, "123sdfsd") - testParseUfloatError(t, "sdsf234") - testParseUfloatError(t, "sdfdf") - - // non-num chars in exponent - testParseUfloatError(t, "123e3s") - testParseUfloatError(t, "12.3e-op") - testParseUfloatError(t, "123E+SS5") - - // duplicate point - testParseUfloatError(t, "1.3.4") - - // duplicate exponent - testParseUfloatError(t, "123e5e6") - - // missing exponent - testParseUfloatError(t, "123534e") -} - -func testParseUfloatError(t *testing.T, s string) { - n, err := ParseUfloat([]byte(s)) - if err == nil { - t.Fatalf("Expecting error when parsing %q. obtained %f", s, n) - } - if n >= 0 { - t.Fatalf("Expecting negative num instead of %f when parsing %q", n, s) - } -} - -func testParseUfloatSuccess(t *testing.T, s string, expectedF float64) { - f, err := ParseUfloat([]byte(s)) - if err != nil { - t.Fatalf("Unexpected error when parsing %q: %v", s, err) - } - delta := f - expectedF - if delta < 0 { - delta = -delta - } - if delta > expectedF*1e-10 { - t.Fatalf("Unexpected value when parsing %q: %f. Expected %f", s, f, expectedF) - } -} - -func testParseUintError(t *testing.T, s string) { - n, err := ParseUint([]byte(s)) - if err == nil { - t.Fatalf("Expecting error when parsing %q. obtained %d", s, n) - } - if n >= 0 { - t.Fatalf("Unexpected n=%d when parsing %q. Expected negative num", n, s) - } -} - -func testParseUintSuccess(t *testing.T, s string, expectedN int) { - n, err := ParseUint([]byte(s)) - if err != nil { - t.Fatalf("Unexpected error when parsing %q: %v", s, err) - } - if n != expectedN { - t.Fatalf("Unexpected value %d. Expected %d. num=%q", n, expectedN, s) - } -} - -func TestAppendUnquotedArg(t *testing.T) { - t.Parallel() - - testAppendUnquotedArg(t, "", "") - testAppendUnquotedArg(t, "abc", "abc") - testAppendUnquotedArg(t, "тест.abc", "тест.abc") - testAppendUnquotedArg(t, "%D1%82%D0%B5%D1%81%D1%82%20%=&;:", "тест %=&;:") -} - -func testAppendUnquotedArg(t *testing.T, s, expectedS string) { - // test appending to nil - result := AppendUnquotedArg(nil, []byte(s)) - if string(result) != expectedS { - t.Fatalf("Unexpected AppendUnquotedArg(%q)=%q, want %q", s, result, expectedS) - } - - // test appending to prefix - prefix := "prefix" - dst := []byte(prefix) - dst = AppendUnquotedArg(dst, []byte(s)) - if !bytes.HasPrefix(dst, []byte(prefix)) { - t.Fatalf("Unexpected prefix for AppendUnquotedArg(%q)=%q, want %q", s, dst, prefix) - } - result = dst[len(prefix):] - if string(result) != expectedS { - t.Fatalf("Unexpected AppendUnquotedArg(%q)=%q, want %q", s, result, expectedS) - } - - // test in-place appending - result = []byte(s) - result = AppendUnquotedArg(result[:0], result) - if string(result) != expectedS { - t.Fatalf("Unexpected AppendUnquotedArg(%q)=%q, want %q", s, result, expectedS) - } - - // verify AppendQuotedArg <-> AppendUnquotedArg conversion - quotedS := AppendQuotedArg(nil, []byte(s)) - unquotedS := AppendUnquotedArg(nil, quotedS) - if s != string(unquotedS) { - t.Fatalf("Unexpected AppendUnquotedArg(AppendQuotedArg(%q))=%q, want %q", s, unquotedS, s) - } -} diff --git a/lib/fasthttp/bytesconv_timing_test.go b/lib/fasthttp/bytesconv_timing_test.go deleted file mode 100644 index c34d7493e..000000000 --- a/lib/fasthttp/bytesconv_timing_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package fasthttp - -import ( - "bufio" - "html" - "net" - "testing" - - "infini.sh/framework/lib/bytebufferpool" -) - -func BenchmarkAppendHTMLEscape(b *testing.B) { - sOrig := "foobarbazxxxyyyzzz" - sExpected := string(AppendHTMLEscape(nil, sOrig)) - b.RunParallel(func(pb *testing.PB) { - var buf []byte - for pb.Next() { - for i := 0; i < 10; i++ { - buf = AppendHTMLEscape(buf[:0], sOrig) - if string(buf) != sExpected { - b.Fatalf("unexpected escaped string: %q. Expecting %q", buf, sExpected) - } - } - } - }) -} - -func BenchmarkHTMLEscapeString(b *testing.B) { - sOrig := "foobarbazxxxyyyzzz" - sExpected := html.EscapeString(sOrig) - b.RunParallel(func(pb *testing.PB) { - var s string - for pb.Next() { - for i := 0; i < 10; i++ { - s = html.EscapeString(sOrig) - if s != sExpected { - b.Fatalf("unexpected escaped string: %q. Expecting %q", s, sExpected) - } - } - } - }) -} - -func BenchmarkParseIPv4(b *testing.B) { - ipStr := []byte("123.145.167.189") - b.RunParallel(func(pb *testing.PB) { - var ip net.IP - var err error - for pb.Next() { - ip, err = ParseIPv4(ip, ipStr) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - } - }) -} - -func BenchmarkAppendIPv4(b *testing.B) { - ip := net.ParseIP("123.145.167.189") - b.RunParallel(func(pb *testing.PB) { - var buf []byte - for pb.Next() { - buf = AppendIPv4(buf[:0], ip) - } - }) -} - -func BenchmarkWriteHexInt(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var w bytebufferpool.ByteBuffer - bw := bufio.NewWriter(&w) - i := 0 - for pb.Next() { - writeHexInt(bw, i) //nolint:errcheck - i++ - if i > 0x7fffffff { - i = 0 - } - w.Reset() - bw.Reset(&w) - } - }) -} - -func BenchmarkParseUint(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - buf := []byte("1234567") - for pb.Next() { - n, err := ParseUint(buf) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - if n != 1234567 { - b.Fatalf("unexpected result: %d. Expecting %q", n, buf) - } - } - }) -} - -func BenchmarkAppendUint(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var buf []byte - i := 0 - for pb.Next() { - buf = AppendUint(buf[:0], i) - i++ - if i > 0x7fffffff { - i = 0 - } - } - }) -} - -func BenchmarkLowercaseBytesNoop(b *testing.B) { - src := []byte("foobarbaz_lowercased_all") - b.RunParallel(func(pb *testing.PB) { - s := make([]byte, len(src)) - for pb.Next() { - copy(s, src) - lowercaseBytes(s) - } - }) -} - -func BenchmarkLowercaseBytesAll(b *testing.B) { - src := []byte("FOOBARBAZ_UPPERCASED_ALL") - b.RunParallel(func(pb *testing.PB) { - s := make([]byte, len(src)) - for pb.Next() { - copy(s, src) - lowercaseBytes(s) - } - }) -} - -func BenchmarkLowercaseBytesMixed(b *testing.B) { - src := []byte("Foobarbaz_Uppercased_Mix") - b.RunParallel(func(pb *testing.PB) { - s := make([]byte, len(src)) - for pb.Next() { - copy(s, src) - lowercaseBytes(s) - } - }) -} - -func BenchmarkAppendUnquotedArgFastPath(b *testing.B) { - src := []byte("foobarbaz no quoted chars fdskjsdf jklsdfdfskljd;aflskjdsaf fdsklj fsdkj fsdl kfjsdlk jfsdklj fsdfsdf sdfkflsd") - b.RunParallel(func(pb *testing.PB) { - var dst []byte - for pb.Next() { - dst = AppendUnquotedArg(dst[:0], src) - } - }) -} - -func BenchmarkAppendUnquotedArgSlowPath(b *testing.B) { - src := []byte("D0%B4%20%D0%B0%D0%B2%D0%BB%D0%B4%D1%84%D1%8B%D0%B0%D0%BE%20%D1%84%D0%B2%D0%B6%D0%BB%D0%B4%D1%8B%20%D0%B0%D0%BE") - b.RunParallel(func(pb *testing.PB) { - var dst []byte - for pb.Next() { - dst = AppendUnquotedArg(dst[:0], src) - } - }) -} diff --git a/lib/fasthttp/client_example_test.go b/lib/fasthttp/client_example_test.go deleted file mode 100644 index 893c786d2..000000000 --- a/lib/fasthttp/client_example_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package fasthttp_test - -import ( - "log" - - "infini.sh/framework/lib/fasthttp" -) - -func ExampleHostClient() { - // Perpare a client, which fetches webpages via HTTP proxy listening - // on the localhost:8080. - c := &fasthttp.HostClient{ - Addr: "localhost:8080", - } - - // Fetch google page via local proxy. - statusCode, body, err := c.Get(nil, "http://google.com/foo/bar") - if err != nil { - log.Fatalf("Error when loading google page through local proxy: %v", err) - } - if statusCode != fasthttp.StatusOK { - log.Fatalf("Unexpected status code: %d. Expecting %d", statusCode, fasthttp.StatusOK) - } - useResponseBody(body) - - // Fetch foobar page via local proxy. Reuse body buffer. - statusCode, body, err = c.Get(body, "http://foobar.com/google/com") - if err != nil { - log.Fatalf("Error when loading foobar page through local proxy: %v", err) - } - if statusCode != fasthttp.StatusOK { - log.Fatalf("Unexpected status code: %d. Expecting %d", statusCode, fasthttp.StatusOK) - } - useResponseBody(body) -} - -func useResponseBody(body []byte) { - // Do something with body :) -} diff --git a/lib/fasthttp/client_test.go b/lib/fasthttp/client_test.go deleted file mode 100644 index 65043efc8..000000000 --- a/lib/fasthttp/client_test.go +++ /dev/null @@ -1,2912 +0,0 @@ -//go:build !ci - -package fasthttp - -import ( - "bufio" - "bytes" - "crypto/tls" - "fmt" - "io" - "net" - "net/url" - "os" - "regexp" - "runtime" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "infini.sh/framework/lib/fasthttp/fasthttputil" -) - -func TestCloseIdleConnections(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - } - go func() { - if err := s.Serve(ln); err != nil { - t.Error(err) - } - }() - - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - - if _, _, err := c.Get(nil, "http://google.com"); err != nil { - t.Fatal(err) - } - - connsLen := func() int { - c.mLock.Lock() - defer c.mLock.Unlock() - - if _, ok := c.m["google.com"]; !ok { - return 0 - } - - c.m["google.com"].connsLock.Lock() - defer c.m["google.com"].connsLock.Unlock() - - return len(c.m["google.com"].conns) - } - - if conns := connsLen(); conns > 1 { - t.Errorf("expected 1 conns got %d", conns) - } - - c.CloseIdleConnections() - - if conns := connsLen(); conns > 0 { - t.Errorf("expected 0 conns got %d", conns) - } -} - -func TestPipelineClientSetUserAgent(t *testing.T) { - t.Parallel() - - testPipelineClientSetUserAgent(t, 0) -} - -func TestPipelineClientSetUserAgentTimeout(t *testing.T) { - t.Parallel() - - testPipelineClientSetUserAgent(t, time.Second) -} - -func testPipelineClientSetUserAgent(t *testing.T, timeout time.Duration) { - ln := fasthttputil.NewInmemoryListener() - - userAgentSeen := "" - s := &Server{ - Handler: func(ctx *RequestCtx) { - userAgentSeen = string(ctx.UserAgent()) - }, - } - go s.Serve(ln) //nolint:errcheck - - userAgent := "I'm not fasthttp" - c := &HostClient{ - Name: userAgent, - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - req := defaultHTTPPool.AcquireRequest() - res := defaultHTTPPool.AcquireResponse() - - req.SetRequestURI("http://example.com") - - var err error - if timeout <= 0 { - err = c.Do(req, res) - } else { - err = c.DoTimeout(req, res, timeout) - } - - if err != nil { - t.Fatal(err) - } - if userAgentSeen != userAgent { - t.Fatalf("User-Agent defers %q != %q", userAgentSeen, userAgent) - } -} - -func TestPipelineClientIssue832(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - req := defaultHTTPPool.AcquireRequest() - // Don't defer ReleaseRequest as we use it in a goroutine that might not be done at the end. - - req.SetHost("example.com") - - res := defaultHTTPPool.AcquireResponse() - // Don't defer ReleaseResponse as we use it in a goroutine that might not be done at the end. - - client := PipelineClient{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - ReadTimeout: time.Millisecond * 10, - Logger: &testLogger{}, // Ignore log output. - } - - attempts := 10 - go func() { - for i := 0; i < attempts; i++ { - c, err := ln.Accept() - if err != nil { - t.Error(err) - } - if c != nil { - go func() { - time.Sleep(time.Millisecond * 50) - c.Close() - }() - } - } - }() - - done := make(chan int) - go func() { - defer close(done) - - for i := 0; i < attempts; i++ { - if err := client.Do(req, res); err == nil { - t.Error("error expected") - } - } - }() - - select { - case <-time.After(time.Second * 2): - t.Fatal("PipelineClient did not restart worker") - case <-done: - } -} - -func TestClientInvalidURI(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - requests := int64(0) - s := &Server{ - Handler: func(_ *RequestCtx) { - atomic.AddInt64(&requests, 1) - }, - } - go s.Serve(ln) //nolint:errcheck - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - req, res := defaultHTTPPool.AcquireRequest(), defaultHTTPPool.AcquireResponse() - defer func() { - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(res) - }() - req.Header.SetMethod(MethodGet) - req.SetRequestURI("http://example.com\r\n\r\nGET /\r\n\r\n") - err := c.Do(req, res) - if err == nil { - t.Fatal("expected error (missing required Host header in request)") - } - if n := atomic.LoadInt64(&requests); n != 0 { - t.Fatalf("0 requests expected, got %d", n) - } -} - -func TestClientGetWithBody(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - body := ctx.Request.Body() - ctx.Write(body) //nolint:errcheck - }, - } - go s.Serve(ln) //nolint:errcheck - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - req, res := defaultHTTPPool.AcquireRequest(), defaultHTTPPool.AcquireResponse() - defer func() { - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(res) - }() - req.Header.SetMethod(MethodGet) - req.SetRequestURI("http://example.com") - req.SetBodyString("test") - err := c.Do(req, res) - if err != nil { - t.Fatal(err) - } - if len(res.Body()) == 0 { - t.Fatal("missing request body") - } -} - -func TestClientURLAuth(t *testing.T) { - t.Parallel() - - cases := map[string]string{ - "user:pass@": "Basic dXNlcjpwYXNz", - "foo:@": "Basic Zm9vOg==", - ":@": "", - "@": "", - "": "", - } - - ch := make(chan string, 1) - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - ch <- string(ctx.Request.Header.Peek(HeaderAuthorization)) - }, - } - go s.Serve(ln) //nolint:errcheck - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - for up, expected := range cases { - req := defaultHTTPPool.AcquireRequest() - req.Header.SetMethod(MethodGet) - req.SetRequestURI("http://" + up + "example.com/foo/bar") - if err := c.Do(req, nil); err != nil { - t.Fatal(err) - } - - val := <-ch - - if val != expected { - t.Fatalf("wrong %q header: %q expected %q", HeaderAuthorization, val, expected) - } - } -} - -func TestClientNilResp(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - } - go s.Serve(ln) //nolint:errcheck - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - req := defaultHTTPPool.AcquireRequest() - req.Header.SetMethod(MethodGet) - req.SetRequestURI("http://example.com") - if err := c.Do(req, nil); err != nil { - t.Fatal(err) - } - if err := c.DoTimeout(req, nil, time.Second); err != nil { - t.Fatal(err) - } - ln.Close() -} - -func TestPipelineClientNilResp(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - } - go s.Serve(ln) //nolint:errcheck - c := &PipelineClient{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - req := defaultHTTPPool.AcquireRequest() - req.Header.SetMethod(MethodGet) - req.SetRequestURI("http://example.com") - if err := c.Do(req, nil); err != nil { - t.Fatal(err) - } - if err := c.DoTimeout(req, nil, time.Second); err != nil { - t.Fatal(err) - } - if err := c.DoDeadline(req, nil, time.Now().Add(time.Second)); err != nil { - t.Fatal(err) - } -} - -func TestClientParseConn(t *testing.T) { - t.Parallel() - - network := "tcp" - ln, _ := net.Listen(network, "127.0.0.1:0") - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - } - go s.Serve(ln) //nolint:errcheck - host := ln.Addr().String() - c := &Client{} - req, res := defaultHTTPPool.AcquireRequest(), defaultHTTPPool.AcquireResponse() - defer func() { - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(res) - }() - req.SetRequestURI("http://" + host + "") - if err := c.Do(req, res); err != nil { - t.Fatal(err) - } - - if res.RemoteAddr().Network() != network { - t.Fatalf("req RemoteAddr parse network fail: %q, hope: %q", res.RemoteAddr().Network(), network) - } - if host != res.RemoteAddr().String() { - t.Fatalf("req RemoteAddr parse addr fail: %q, hope: %q", res.RemoteAddr().String(), host) - } - - if !regexp.MustCompile(`^127\.0\.0\.1:[0-9]{4,5}$`).MatchString(res.LocalAddr().String()) { - t.Fatalf("res LocalAddr addr match fail: %q, hope match: %q", res.LocalAddr().String(), "^127.0.0.1:[0-9]{4,5}$") - } -} - -func TestClientPostArgs(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - body := ctx.Request.Body() - if len(body) == 0 { - return - } - ctx.Write(body) //nolint:errcheck - }, - } - go s.Serve(ln) //nolint:errcheck - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - req, res := defaultHTTPPool.AcquireRequest(), defaultHTTPPool.AcquireResponse() - defer func() { - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(res) - }() - args := req.PostArgs() - args.Add("addhttp2", "support") - args.Add("fast", "http") - req.Header.SetMethod(MethodPost) - req.SetRequestURI("http://make.fasthttp.great?again") - err := c.Do(req, res) - if err != nil { - t.Fatal(err) - } - if len(res.Body()) == 0 { - t.Fatal("cannot set args as body") - } -} - -func TestClientRedirectSameSchema(t *testing.T) { - t.Parallel() - - listenHTTPS1 := testClientRedirectListener(t, true) - defer listenHTTPS1.Close() - - listenHTTPS2 := testClientRedirectListener(t, true) - defer listenHTTPS2.Close() - - sHTTPS1 := testClientRedirectChangingSchemaServer(t, listenHTTPS1, listenHTTPS1, true) - defer sHTTPS1.Stop() - - sHTTPS2 := testClientRedirectChangingSchemaServer(t, listenHTTPS2, listenHTTPS2, false) - defer sHTTPS2.Stop() - - destURL := fmt.Sprintf("https://%s/baz", listenHTTPS1.Addr().String()) - - urlParsed, err := url.Parse(destURL) - if err != nil { - t.Fatal(err) - return - } - - reqClient := &HostClient{ - IsTLS: true, - Addr: urlParsed.Host, - TLSConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - - statusCode, _, err := reqClient.GetTimeout(nil, destURL, 4000*time.Millisecond) - if err != nil { - t.Fatalf("HostClient error: %v", err) - return - } - - if statusCode != 200 { - t.Fatalf("HostClient error code response %d", statusCode) - return - } -} - -func TestClientRedirectClientChangingSchemaHttp2Https(t *testing.T) { - t.Parallel() - - listenHTTPS := testClientRedirectListener(t, true) - defer listenHTTPS.Close() - - listenHTTP := testClientRedirectListener(t, false) - defer listenHTTP.Close() - - sHTTPS := testClientRedirectChangingSchemaServer(t, listenHTTPS, listenHTTP, true) - defer sHTTPS.Stop() - - sHTTP := testClientRedirectChangingSchemaServer(t, listenHTTPS, listenHTTP, false) - defer sHTTP.Stop() - - destURL := fmt.Sprintf("http://%s/baz", listenHTTP.Addr().String()) - - reqClient := &Client{ - TLSConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - - statusCode, _, err := reqClient.GetTimeout(nil, destURL, 4000*time.Millisecond) - if err != nil { - t.Fatalf("HostClient error: %v", err) - return - } - - if statusCode != 200 { - t.Fatalf("HostClient error code response %d", statusCode) - return - } -} - -func TestClientRedirectHostClientChangingSchemaHttp2Https(t *testing.T) { - t.Parallel() - - listenHTTPS := testClientRedirectListener(t, true) - defer listenHTTPS.Close() - - listenHTTP := testClientRedirectListener(t, false) - defer listenHTTP.Close() - - sHTTPS := testClientRedirectChangingSchemaServer(t, listenHTTPS, listenHTTP, true) - defer sHTTPS.Stop() - - sHTTP := testClientRedirectChangingSchemaServer(t, listenHTTPS, listenHTTP, false) - defer sHTTP.Stop() - - destURL := fmt.Sprintf("http://%s/baz", listenHTTP.Addr().String()) - - urlParsed, err := url.Parse(destURL) - if err != nil { - t.Fatal(err) - return - } - - reqClient := &HostClient{ - Addr: urlParsed.Host, - TLSConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - - _, _, err = reqClient.GetTimeout(nil, destURL, 4000*time.Millisecond) - if err != ErrHostClientRedirectToDifferentScheme { - t.Fatal("expected HostClient error") - } -} - -func testClientRedirectListener(t *testing.T, isTLS bool) net.Listener { - var ln net.Listener - var err error - var tlsConfig *tls.Config - - if isTLS { - certData, keyData, kerr := GenerateTestCertificate("localhost") - if kerr != nil { - t.Fatal(kerr) - } - - cert, kerr := tls.X509KeyPair(certData, keyData) - if kerr != nil { - t.Fatal(kerr) - } - - tlsConfig = &tls.Config{ - Certificates: []tls.Certificate{cert}, - } - ln, err = tls.Listen("tcp", "localhost:0", tlsConfig) - } else { - ln, err = net.Listen("tcp", "localhost:0") - } - - if err != nil { - t.Fatalf("cannot listen isTLS %v: %v", isTLS, err) - } - - return ln -} - -func testClientRedirectChangingSchemaServer(t *testing.T, https, http net.Listener, isTLS bool) *testEchoServer { - s := &Server{ - Handler: func(ctx *RequestCtx) { - if ctx.IsTLS() { - ctx.SetStatusCode(200) - } else { - ctx.Redirect(fmt.Sprintf("https://%s/baz", https.Addr().String()), 301) - } - }, - } - - var ln net.Listener - if isTLS { - ln = https - } else { - ln = http - } - - ch := make(chan struct{}) - go func() { - err := s.Serve(ln) - if err != nil { - t.Errorf("unexpected error returned from Serve(): %v", err) - } - close(ch) - }() - return &testEchoServer{ - s: s, - ln: ln, - ch: ch, - t: t, - } -} - -func TestClientHeaderCase(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - defer ln.Close() - - go func() { - c, err := ln.Accept() - if err != nil { - t.Error(err) - } - c.Write([]byte("HTTP/1.1 200 OK\r\n" + //nolint:errcheck - "content-type: text/plain\r\n" + - "transfer-encoding: chunked\r\n\r\n" + - "24\r\nThis is the data in the first chunk \r\n" + - "1B\r\nand this is the second one \r\n" + - "0\r\n\r\n", - )) - }() - - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - ReadTimeout: time.Millisecond * 10, - - // Even without name normalizing we should parse headers correctly. - DisableHeaderNamesNormalizing: true, - } - - code, body, err := c.Get(nil, "http://example.com") - if err != nil { - t.Error(err) - } else if code != 200 { - t.Errorf("expected status code 200 got %d", code) - } else if string(body) != "This is the data in the first chunk and this is the second one " { - t.Errorf("wrong body: %q", body) - } -} - -func TestClientReadTimeout(t *testing.T) { - if runtime.GOOS == "windows" { - t.SkipNow() - } - - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - timeout := false - s := &Server{ - Handler: func(_ *RequestCtx) { - if timeout { - time.Sleep(time.Second) - } else { - timeout = true - } - }, - Logger: &testLogger{}, // Don't print closed pipe errors. - } - go s.Serve(ln) //nolint:errcheck - - c := &HostClient{ - ReadTimeout: time.Millisecond * 400, - MaxIdemponentCallAttempts: 1, - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - - req := defaultHTTPPool.AcquireRequest() - res := defaultHTTPPool.AcquireResponse() - - req.SetRequestURI("http://localhost") - - // Setting Connection: Close will make the connection be - // returned to the pool. - req.SetConnectionClose() - - if err := c.Do(req, res); err != nil { - t.Fatal(err) - } - - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(res) - - done := make(chan struct{}) - go func() { - req := defaultHTTPPool.AcquireRequest() - res := defaultHTTPPool.AcquireResponse() - - req.SetRequestURI("http://localhost") - req.SetConnectionClose() - - if err := c.Do(req, res); err != ErrTimeout { - t.Errorf("expected ErrTimeout got %#v", err) - } - - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(res) - close(done) - }() - - select { - case <-done: - // This shouldn't take longer than the timeout times the number of requests it is going to try to do. - // Give it an extra second just to be sure. - case <-time.After(c.ReadTimeout*time.Duration(c.MaxIdemponentCallAttempts) + time.Second): - t.Fatal("Client.ReadTimeout didn't work") - } -} - -func TestClientDefaultUserAgent(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - userAgentSeen := "" - s := &Server{ - Handler: func(ctx *RequestCtx) { - userAgentSeen = string(ctx.UserAgent()) - }, - } - go s.Serve(ln) //nolint:errcheck - - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - req := defaultHTTPPool.AcquireRequest() - res := defaultHTTPPool.AcquireResponse() - - req.SetRequestURI("http://example.com") - - err := c.Do(req, res) - if err != nil { - t.Fatal(err) - } - if userAgentSeen != string(defaultUserAgent) { - t.Fatalf("User-Agent defers %q != %q", userAgentSeen, defaultUserAgent) - } -} - -func TestClientSetUserAgent(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - userAgentSeen := "" - s := &Server{ - Handler: func(ctx *RequestCtx) { - userAgentSeen = string(ctx.UserAgent()) - }, - } - go s.Serve(ln) //nolint:errcheck - - userAgent := "I'm not fasthttp" - c := &Client{ - Name: userAgent, - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - req := defaultHTTPPool.AcquireRequest() - res := defaultHTTPPool.AcquireResponse() - - req.SetRequestURI("http://example.com") - - err := c.Do(req, res) - if err != nil { - t.Fatal(err) - } - if userAgentSeen != userAgent { - t.Fatalf("User-Agent defers %q != %q", userAgentSeen, userAgent) - } -} - -func TestClientNoUserAgent(t *testing.T) { - ln := fasthttputil.NewInmemoryListener() - - userAgentSeen := "" - s := &Server{ - Handler: func(ctx *RequestCtx) { - userAgentSeen = string(ctx.UserAgent()) - }, - } - go s.Serve(ln) //nolint:errcheck - - c := &Client{ - NoDefaultUserAgentHeader: true, - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - req := defaultHTTPPool.AcquireRequest() - res := defaultHTTPPool.AcquireResponse() - - req.SetRequestURI("http://example.com") - - err := c.Do(req, res) - if err != nil { - t.Fatal(err) - } - if userAgentSeen != "" { - t.Fatalf("User-Agent wrong %q != %q", userAgentSeen, "") - } -} - -func TestClientDoWithCustomHeaders(t *testing.T) { - t.Parallel() - - // make sure that the client sends all the request headers and body. - ln := fasthttputil.NewInmemoryListener() - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - - uri := "/foo/bar/baz?a=b&cd=12" - headers := map[string]string{ - "Foo": "bar", - "Host": "example.com", - "Content-Type": "asdfsdf", - "a-b-c-d-f": "", - } - body := "request body" - - ch := make(chan error) - go func() { - conn, err := ln.Accept() - if err != nil { - ch <- fmt.Errorf("cannot accept client connection: %w", err) - return - } - br := bufio.NewReader(conn) - - var req Request - if err = req.Read(br); err != nil { - ch <- fmt.Errorf("cannot read client request: %w", err) - return - } - if string(req.Header.Method()) != MethodPost { - ch <- fmt.Errorf("unexpected request method: %q. Expecting %q", req.Header.Method(), MethodPost) - return - } - reqURI := req.RequestURI() - if string(reqURI) != uri { - ch <- fmt.Errorf("unexpected request uri: %q. Expecting %q", reqURI, uri) - return - } - for k, v := range headers { - hv := req.Header.Peek(k) - if string(hv) != v { - ch <- fmt.Errorf("unexpected value for header %q: %q. Expecting %q", k, hv, v) - return - } - } - cl := req.Header.ContentLength() - if cl != len(body) { - ch <- fmt.Errorf("unexpected content-length %d. Expecting %d", cl, len(body)) - return - } - reqBody := req.Body() - if string(reqBody) != body { - ch <- fmt.Errorf("unexpected request body: %q. Expecting %q", reqBody, body) - return - } - - var resp Response - bw := bufio.NewWriter(conn) - if err = resp.Write(bw); err != nil { - ch <- fmt.Errorf("cannot send response: %w", err) - return - } - if err = bw.Flush(); err != nil { - ch <- fmt.Errorf("cannot flush response: %w", err) - return - } - - ch <- nil - }() - - var req Request - req.Header.SetMethod(MethodPost) - req.SetRequestURI(uri) - for k, v := range headers { - req.Header.Set(k, v) - } - req.SetBodyString(body) - - var resp Response - - err := c.DoTimeout(&req, &resp, time.Second) - if err != nil { - t.Fatalf("error when doing request: %v", err) - } - - select { - case <-ch: - case <-time.After(5 * time.Second): - t.Fatalf("timeout") - } -} - -func TestPipelineClientDoSerial(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - testPipelineClientDoConcurrent(t, 1, 0, 0) -} - -func TestPipelineClientDoConcurrent(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - testPipelineClientDoConcurrent(t, 10, 0, 1) -} - -func TestPipelineClientDoBatchDelayConcurrent(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - testPipelineClientDoConcurrent(t, 10, 5*time.Millisecond, 1) -} - -func TestPipelineClientDoBatchDelayConcurrentMultiConn(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - testPipelineClientDoConcurrent(t, 10, 5*time.Millisecond, 3) -} - -func testPipelineClientDoConcurrent(t *testing.T, concurrency int, maxBatchDelay time.Duration, maxConns int) { - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.WriteString("OK") //nolint:errcheck - }, - } - - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &PipelineClient{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - MaxConns: maxConns, - MaxPendingRequests: concurrency, - MaxBatchDelay: maxBatchDelay, - Logger: &testLogger{}, - } - - clientStopCh := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - testPipelineClientDo(t, c) - clientStopCh <- struct{}{} - }() - } - - for i := 0; i < concurrency; i++ { - select { - case <-clientStopCh: - case <-time.After(3 * time.Second): - t.Fatalf("timeout") - } - } - - if c.PendingRequests() != 0 { - t.Fatalf("unexpected number of pending requests: %d. Expecting zero", c.PendingRequests()) - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } -} - -func testPipelineClientDo(t *testing.T, c *PipelineClient) { - var err error - req := defaultHTTPPool.AcquireRequest() - req.SetRequestURI("http://foobar/baz") - resp := defaultHTTPPool.AcquireResponse() - for i := 0; i < 10; i++ { - if i&1 == 0 { - err = c.DoTimeout(req, resp, time.Second) - } else { - err = c.Do(req, resp) - } - if err != nil { - if err == ErrPipelineOverflow { - time.Sleep(10 * time.Millisecond) - continue - } - t.Errorf("unexpected error on iteration %d: %v", i, err) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - body := string(resp.Body()) - if body != "OK" { - t.Errorf("unexpected body: %q. Expecting %q", body, "OK") - } - - // sleep for a while, so the connection to the host may expire. - if i%5 == 0 { - time.Sleep(30 * time.Millisecond) - } - } - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(resp) -} - -func TestPipelineClientDoDisableHeaderNamesNormalizing(t *testing.T) { - t.Parallel() - - testPipelineClientDisableHeaderNamesNormalizing(t, 0) -} - -func TestPipelineClientDoTimeoutDisableHeaderNamesNormalizing(t *testing.T) { - t.Parallel() - - testPipelineClientDisableHeaderNamesNormalizing(t, time.Second) -} - -func testPipelineClientDisableHeaderNamesNormalizing(t *testing.T, timeout time.Duration) { - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Response.Header.Set("foo-BAR", "baz") - }, - DisableHeaderNamesNormalizing: true, - } - - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &PipelineClient{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - DisableHeaderNamesNormalizing: true, - } - - var req Request - req.SetRequestURI("http://aaaai.com/bsdf?sddfsd") - var resp Response - for i := 0; i < 5; i++ { - if timeout > 0 { - if err := c.DoTimeout(&req, &resp, timeout); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } else { - if err := c.Do(&req, &resp); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - hv := resp.Header.Peek("foo-BAR") - if string(hv) != "baz" { - t.Fatalf("unexpected header value: %q. Expecting %q", hv, "baz") - } - hv = resp.Header.Peek("Foo-Bar") - if len(hv) > 0 { - t.Fatalf("unexpected non-empty header value %q", hv) - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } -} - -func TestClientDoTimeoutDisableHeaderNamesNormalizing(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Response.Header.Set("foo-BAR", "baz") - }, - DisableHeaderNamesNormalizing: true, - } - - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - DisableHeaderNamesNormalizing: true, - } - - var req Request - req.SetRequestURI("http://aaaai.com/bsdf?sddfsd") - var resp Response - for i := 0; i < 5; i++ { - if err := c.DoTimeout(&req, &resp, time.Second); err != nil { - t.Fatalf("unexpected error: %v", err) - } - hv := resp.Header.Peek("foo-BAR") - if string(hv) != "baz" { - t.Fatalf("unexpected header value: %q. Expecting %q", hv, "baz") - } - hv = resp.Header.Peek("Foo-Bar") - if len(hv) > 0 { - t.Fatalf("unexpected non-empty header value %q", hv) - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } -} - -func TestClientDoTimeoutDisablePathNormalizing(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - uri := ctx.PhantomURI() - uri.DisablePathNormalizing = true - ctx.Response.Header.Set("received-uri", string(uri.FullURI())) - }, - } - - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - DisablePathNormalizing: true, - } - - urlWithEncodedPath := "http://example.com/encoded/Y%2BY%2FY%3D/stuff" - - var req Request - req.SetRequestURI(urlWithEncodedPath) - var resp Response - for i := 0; i < 5; i++ { - if err := c.DoTimeout(&req, &resp, time.Second); err != nil { - t.Fatalf("unexpected error: %v", err) - } - hv := resp.Header.Peek("received-uri") - if string(hv) != urlWithEncodedPath { - t.Fatalf("request uri was normalized: %q. Expecting %q", hv, urlWithEncodedPath) - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } -} - -func TestHostClientPendingRequests(t *testing.T) { - t.Parallel() - - const concurrency = 10 - doneCh := make(chan struct{}) - readyCh := make(chan struct{}, concurrency) - s := &Server{ - Handler: func(_ *RequestCtx) { - readyCh <- struct{}{} - <-doneCh - }, - } - ln := fasthttputil.NewInmemoryListener() - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &HostClient{ - Addr: "foobar", - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - - pendingRequests := c.PendingRequests() - if pendingRequests != 0 { - t.Fatalf("non-zero pendingRequests: %d", pendingRequests) - } - - resultCh := make(chan error, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - req := defaultHTTPPool.AcquireRequest() - req.SetRequestURI("http://foobar/baz") - resp := defaultHTTPPool.AcquireResponse() - - if err := c.DoTimeout(req, resp, 10*time.Second); err != nil { - resultCh <- fmt.Errorf("unexpected error: %w", err) - return - } - - if resp.StatusCode() != StatusOK { - resultCh <- fmt.Errorf("unexpected status code %d. Expecting %d", resp.StatusCode(), StatusOK) - return - } - resultCh <- nil - }() - } - - // wait while all the requests reach server - for i := 0; i < concurrency; i++ { - select { - case <-readyCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } - - pendingRequests = c.PendingRequests() - if pendingRequests != concurrency { - t.Fatalf("unexpected pendingRequests: %d. Expecting %d", pendingRequests, concurrency) - } - - // unblock request handlers on the server and wait until all the requests are finished. - close(doneCh) - for i := 0; i < concurrency; i++ { - select { - case err := <-resultCh: - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } - - pendingRequests = c.PendingRequests() - if pendingRequests != 0 { - t.Fatalf("non-zero pendingRequests: %d", pendingRequests) - } - - // stop the server - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } -} - -func TestHostClientMaxConnsWithDeadline(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var ( - emptyBodyCount uint8 - ln = fasthttputil.NewInmemoryListener() - timeout = 200 * time.Millisecond - wg sync.WaitGroup - ) - - s := &Server{ - Handler: func(ctx *RequestCtx) { - if len(ctx.PostBody()) == 0 { - emptyBodyCount++ - } - - ctx.WriteString("foo") //nolint:errcheck - }, - } - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &HostClient{ - Addr: "foobar", - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - MaxConns: 1, - } - - for i := 0; i < 5; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - req := defaultHTTPPool.AcquireRequest() - req.SetRequestURI("http://foobar/baz") - req.Header.SetMethod(MethodPost) - req.SetBodyString("bar") - resp := defaultHTTPPool.AcquireResponse() - - for { - if err := c.DoDeadline(req, resp, time.Now().Add(timeout)); err != nil { - if err == ErrNoFreeConns { - time.Sleep(time.Millisecond) - continue - } - t.Errorf("unexpected error: %v", err) - } - break - } - - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code %d. Expecting %d", resp.StatusCode(), StatusOK) - } - - body := resp.Body() - if string(body) != "foo" { - t.Errorf("unexpected body %q. Expecting %q", body, "abcd") - } - }() - } - wg.Wait() - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - - if emptyBodyCount > 0 { - t.Fatalf("at least one request body was empty") - } -} - -func TestHostClientMaxConnDuration(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - connectionCloseCount := uint32(0) - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.WriteString("abcd") //nolint:errcheck - if ctx.Request.ConnectionClose() { - atomic.AddUint32(&connectionCloseCount, 1) - } - }, - } - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &HostClient{ - Addr: "foobar", - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - MaxConnDuration: 10 * time.Millisecond, - } - - for i := 0; i < 5; i++ { - statusCode, body, err := c.Get(nil, "http://aaaa.com/bbb/cc") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if statusCode != StatusOK { - t.Fatalf("unexpected status code %d. Expecting %d", statusCode, StatusOK) - } - if string(body) != "abcd" { - t.Fatalf("unexpected body %q. Expecting %q", body, "abcd") - } - time.Sleep(c.MaxConnDuration) - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - - if connectionCloseCount == 0 { - t.Fatalf("expecting at least one 'Connection: close' request header") - } -} - -func TestHostClientMultipleAddrs(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Write(ctx.Host()) //nolint:errcheck - ctx.SetConnectionClose() - }, - } - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - dialsCount := make(map[string]int) - c := &HostClient{ - Addr: "foo,bar,baz", - Dial: func(addr string) (net.Conn, error) { - dialsCount[addr]++ - return ln.Dial() - }, - } - - for i := 0; i < 9; i++ { - statusCode, body, err := c.Get(nil, "http://foobar/baz/aaa?bbb=ddd") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if statusCode != StatusOK { - t.Fatalf("unexpected status code %d. Expecting %d", statusCode, StatusOK) - } - if string(body) != "foobar" { - t.Fatalf("unexpected body %q. Expecting %q", body, "foobar") - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - - if len(dialsCount) != 3 { - t.Fatalf("unexpected dialsCount size %d. Expecting 3", len(dialsCount)) - } - for _, k := range []string{"foo", "bar", "baz"} { - if dialsCount[k] != 3 { - t.Fatalf("unexpected dialsCount for %q. Expecting 3", k) - } - } -} - -func TestClientFollowRedirects(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - switch string(ctx.Path()) { - case "/foo": - u := ctx.PhantomURI() - u.Update("/xy?z=wer") - ctx.Redirect(u.String(), StatusFound) - case "/xy": - u := ctx.PhantomURI() - u.Update("/bar") - ctx.Redirect(u.String(), StatusFound) - default: - ctx.Success("text/plain", ctx.Path()) - } - }, - } - ln := fasthttputil.NewInmemoryListener() - - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &HostClient{ - Addr: "xxx", - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - } - - for i := 0; i < 10; i++ { - statusCode, body, err := c.GetTimeout(nil, "http://xxx/foo", time.Second) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if statusCode != StatusOK { - t.Fatalf("unexpected status code: %d", statusCode) - } - if string(body) != "/bar" { - t.Fatalf("unexpected response %q. Expecting %q", body, "/bar") - } - } - - for i := 0; i < 10; i++ { - statusCode, body, err := c.Get(nil, "http://xxx/aaab/sss") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if statusCode != StatusOK { - t.Fatalf("unexpected status code: %d", statusCode) - } - if string(body) != "/aaab/sss" { - t.Fatalf("unexpected response %q. Expecting %q", body, "/aaab/sss") - } - } - - for i := 0; i < 10; i++ { - req := defaultHTTPPool.AcquireRequest() - resp := defaultHTTPPool.AcquireResponse() - - req.SetRequestURI("http://xxx/foo") - - err := c.DoRedirects(req, resp, 16) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if statusCode := resp.StatusCode(); statusCode != StatusOK { - t.Fatalf("unexpected status code: %d", statusCode) - } - - if body := string(resp.Body()); body != "/bar" { - t.Fatalf("unexpected response %q. Expecting %q", body, "/bar") - } - - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(resp) - } - - req := defaultHTTPPool.AcquireRequest() - resp := defaultHTTPPool.AcquireResponse() - - req.SetRequestURI("http://xxx/foo") - - err := c.DoRedirects(req, resp, 0) - if have, want := err, ErrTooManyRedirects; have != want { - t.Fatalf("want error: %v, have %v", want, have) - } - - defaultHTTPPool.ReleaseRequest(req) - defaultHTTPPool.ReleaseResponse(resp) -} - -func TestClientGetTimeoutSuccess(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - testClientGetTimeoutSuccess(t, &defaultClient, "http://"+s.Addr(), 100) -} - -func TestClientGetTimeoutSuccessConcurrent(t *testing.T) { - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - testClientGetTimeoutSuccess(t, &defaultClient, "http://"+s.Addr(), 100) - }() - } - wg.Wait() -} - -func TestClientDoTimeoutSuccess(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - testClientDoTimeoutSuccess(t, &defaultClient, "http://"+s.Addr(), 100) -} - -func TestClientDoTimeoutSuccessConcurrent(t *testing.T) { - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - testClientDoTimeoutSuccess(t, &defaultClient, "http://"+s.Addr(), 100) - }() - } - wg.Wait() -} - -func TestClientGetTimeoutError(t *testing.T) { - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - testConn, _ := net.Dial("tcp", s.ln.Addr().String()) - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return &readTimeoutConn{Conn: testConn, t: time.Second}, nil - }, - } - - testClientGetTimeoutError(t, c, 100) -} - -func TestClientGetTimeoutErrorConcurrent(t *testing.T) { - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - testConn, _ := net.Dial("tcp", s.ln.Addr().String()) - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return &readTimeoutConn{Conn: testConn, t: time.Second}, nil - }, - MaxConnsPerHost: 1000, - } - - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - testClientGetTimeoutError(t, c, 100) - }() - } - wg.Wait() -} - -func TestClientDoTimeoutError(t *testing.T) { - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - testConn, _ := net.Dial("tcp", s.ln.Addr().String()) - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return &readTimeoutConn{Conn: testConn, t: time.Second}, nil - }, - } - - testClientDoTimeoutError(t, c, 100) -} - -func TestClientDoTimeoutErrorConcurrent(t *testing.T) { - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - testConn, _ := net.Dial("tcp", s.ln.Addr().String()) - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return &readTimeoutConn{Conn: testConn, t: time.Second}, nil - }, - MaxConnsPerHost: 1000, - } - - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - testClientDoTimeoutError(t, c, 100) - }() - } - wg.Wait() -} - -func testClientDoTimeoutError(t *testing.T, c *Client, n int) { - var req Request - var resp Response - req.SetRequestURI("http://foobar.com/baz") - for i := 0; i < n; i++ { - err := c.DoTimeout(&req, &resp, time.Millisecond) - if err == nil { - t.Errorf("expecting error") - } - if err != ErrTimeout { - t.Errorf("unexpected error: %v. Expecting %v", err, ErrTimeout) - } - } -} - -func testClientGetTimeoutError(t *testing.T, c *Client, n int) { - buf := make([]byte, 10) - for i := 0; i < n; i++ { - statusCode, body, err := c.GetTimeout(buf, "http://foobar.com/baz", time.Millisecond) - if err == nil { - t.Errorf("expecting error") - } - if err != ErrTimeout { - t.Errorf("unexpected error: %v. Expecting %v", err, ErrTimeout) - } - if statusCode != 0 { - t.Errorf("unexpected statusCode=%d. Expecting %d", statusCode, 0) - } - if body == nil { - t.Errorf("body must be non-nil") - } - } -} - -type readTimeoutConn struct { - net.Conn - t time.Duration - wc chan struct{} - rc chan struct{} -} - -func (r *readTimeoutConn) Read(p []byte) (int, error) { - <-r.rc - return 0, os.ErrDeadlineExceeded -} - -func (r *readTimeoutConn) Write(p []byte) (int, error) { - <-r.wc - return 0, os.ErrDeadlineExceeded -} - -func (r *readTimeoutConn) Close() error { - return nil -} - -func (r *readTimeoutConn) LocalAddr() net.Addr { - return nil -} - -func (r *readTimeoutConn) RemoteAddr() net.Addr { - return nil -} - -func (r *readTimeoutConn) SetReadDeadline(d time.Time) error { - r.rc = make(chan struct{}, 1) - go func() { - time.Sleep(time.Until(d)) - r.rc <- struct{}{} - }() - return nil -} - -func (r *readTimeoutConn) SetWriteDeadline(d time.Time) error { - r.wc = make(chan struct{}, 1) - go func() { - time.Sleep(time.Until(d)) - r.wc <- struct{}{} - }() - return nil -} - -func TestClientNonIdempotentRetry(t *testing.T) { - t.Parallel() - - dialsCount := 0 - c := &Client{ - Dial: func(_ string) (net.Conn, error) { - dialsCount++ - switch dialsCount { - case 1, 2: - return &readErrorConn{}, nil - case 3: - return &singleReadConn{ - s: "HTTP/1.1 345 OK\r\nContent-Type: foobar\r\nContent-Length: 7\r\n\r\n0123456", - }, nil - default: - t.Fatalf("unexpected number of dials: %d", dialsCount) - } - panic("unreachable") - }, - } - - // This POST must succeed, since the readErrorConn closes - // the connection before sending any response. - // So the client must retry non-idempotent request. - dialsCount = 0 - statusCode, body, err := c.Post(nil, "http://foobar/a/b", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if statusCode != 345 { - t.Fatalf("unexpected status code: %d. Expecting 345", statusCode) - } - if string(body) != "0123456" { - t.Fatalf("unexpected body: %q. Expecting %q", body, "0123456") - } - - // Verify that idempotent GET succeeds. - dialsCount = 0 - statusCode, body, err = c.Get(nil, "http://foobar/a/b") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if statusCode != 345 { - t.Fatalf("unexpected status code: %d. Expecting 345", statusCode) - } - if string(body) != "0123456" { - t.Fatalf("unexpected body: %q. Expecting %q", body, "0123456") - } -} - -func TestClientNonIdempotentRetry_BodyStream(t *testing.T) { - t.Parallel() - - dialsCount := 0 - c := &Client{ - Dial: func(_ string) (net.Conn, error) { - dialsCount++ - switch dialsCount { - case 1, 2: - return &readErrorConn{}, nil - case 3: - return &singleEchoConn{ - b: []byte("HTTP/1.1 345 OK\r\nContent-Type: foobar\r\n\r\n"), - }, nil - default: - t.Fatalf("unexpected number of dials: %d", dialsCount) - } - panic("unreachable") - }, - } - - dialsCount = 0 - - req := Request{} - res := Response{} - - req.SetRequestURI("http://foobar/a/b") - req.Header.SetMethod("POST") - body := bytes.NewBufferString("test") - req.SetBodyStream(body, body.Len()) - - err := c.Do(&req, &res) - if err == nil { - t.Fatal("expected error from being unable to retry a bodyStream") - } -} - -func TestClientIdempotentRequest(t *testing.T) { - t.Parallel() - - dialsCount := 0 - c := &Client{ - Dial: func(_ string) (net.Conn, error) { - dialsCount++ - switch dialsCount { - case 1: - return &singleReadConn{ - s: "invalid response", - }, nil - case 2: - return &writeErrorConn{}, nil - case 3: - return &readErrorConn{}, nil - case 4: - return &singleReadConn{ - s: "HTTP/1.1 345 OK\r\nContent-Type: foobar\r\nContent-Length: 7\r\n\r\n0123456", - }, nil - default: - t.Fatalf("unexpected number of dials: %d", dialsCount) - } - panic("unreachable") - }, - } - - // idempotent GET must succeed. - statusCode, body, err := c.Get(nil, "http://foobar/a/b") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if statusCode != 345 { - t.Fatalf("unexpected status code: %d. Expecting 345", statusCode) - } - if string(body) != "0123456" { - t.Fatalf("unexpected body: %q. Expecting %q", body, "0123456") - } - - var args Args - - // non-idempotent POST must fail on incorrect singleReadConn - dialsCount = 0 - _, _, err = c.Post(nil, "http://foobar/a/b", &args) - if err == nil { - t.Fatalf("expecting error") - } - - // non-idempotent POST must fail on incorrect singleReadConn - dialsCount = 0 - _, _, err = c.Post(nil, "http://foobar/a/b", nil) - if err == nil { - t.Fatalf("expecting error") - } -} - -func TestClientRetryRequestWithCustomDecider(t *testing.T) { - t.Parallel() - - dialsCount := 0 - c := &Client{ - Dial: func(_ string) (net.Conn, error) { - dialsCount++ - switch dialsCount { - case 1: - return &singleReadConn{ - s: "invalid response", - }, nil - case 2: - return &writeErrorConn{}, nil - case 3: - return &readErrorConn{}, nil - case 4: - return &singleReadConn{ - s: "HTTP/1.1 345 OK\r\nContent-Type: foobar\r\nContent-Length: 7\r\n\r\n0123456", - }, nil - default: - t.Fatalf("unexpected number of dials: %d", dialsCount) - } - panic("unreachable") - }, - RetryIf: func(req *Request) bool { - return req.PhantomURI().String() == "http://foobar/a/b" - }, - } - - var args Args - - // Post must succeed for http://foobar/a/b uri. - statusCode, body, err := c.Post(nil, "http://foobar/a/b", &args) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if statusCode != 345 { - t.Fatalf("unexpected status code: %d. Expecting 345", statusCode) - } - if string(body) != "0123456" { - t.Fatalf("unexpected body: %q. Expecting %q", body, "0123456") - } - - // POST must fail for http://foobar/a/b/c uri. - dialsCount = 0 - _, _, err = c.Post(nil, "http://foobar/a/b/c", &args) - if err == nil { - t.Fatalf("expecting error") - } -} - -func TestHostClientTransport(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.WriteString("abcd") //nolint:errcheck - }, - } - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &HostClient{ - Addr: "foobar", - Transport: func() TransportFunc { - c, _ := ln.Dial() - - br := bufio.NewReader(c) - bw := bufio.NewWriter(c) - - return func(req *Request, res *Response) error { - if err := req.Write(bw); err != nil { - return err - } - - if err := bw.Flush(); err != nil { - return err - } - - return res.Read(br) - } - }(), - } - - for i := 0; i < 5; i++ { - statusCode, body, err := c.Get(nil, "http://aaaa.com/bbb/cc") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if statusCode != StatusOK { - t.Fatalf("unexpected status code %d. Expecting %d", statusCode, StatusOK) - } - if string(body) != "abcd" { - t.Fatalf("unexpected body %q. Expecting %q", body, "abcd") - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } -} - -type writeErrorConn struct { - net.Conn -} - -func (w *writeErrorConn) Write(p []byte) (int, error) { - return 1, fmt.Errorf("error") -} - -func (w *writeErrorConn) Close() error { - return nil -} - -func (w *writeErrorConn) LocalAddr() net.Addr { - return nil -} - -func (w *writeErrorConn) RemoteAddr() net.Addr { - return nil -} - -type readErrorConn struct { - net.Conn -} - -func (r *readErrorConn) Read(p []byte) (int, error) { - return 0, fmt.Errorf("error") -} - -func (r *readErrorConn) Write(p []byte) (int, error) { - return len(p), nil -} - -func (r *readErrorConn) Close() error { - return nil -} - -func (r *readErrorConn) LocalAddr() net.Addr { - return nil -} - -func (r *readErrorConn) RemoteAddr() net.Addr { - return nil -} - -type singleReadConn struct { - net.Conn - s string - n int -} - -func (r *singleReadConn) Read(p []byte) (int, error) { - if len(r.s) == r.n { - return 0, io.EOF - } - n := copy(p, []byte(r.s[r.n:])) - r.n += n - return n, nil -} - -func (r *singleReadConn) Write(p []byte) (int, error) { - return len(p), nil -} - -func (r *singleReadConn) Close() error { - return nil -} - -func (r *singleReadConn) LocalAddr() net.Addr { - return nil -} - -func (r *singleReadConn) RemoteAddr() net.Addr { - return nil -} - -type singleEchoConn struct { - net.Conn - b []byte - n int -} - -func (r *singleEchoConn) Read(p []byte) (int, error) { - if len(r.b) == r.n { - return 0, io.EOF - } - n := copy(p, r.b[r.n:]) - r.n += n - return n, nil -} - -func (r *singleEchoConn) Write(p []byte) (int, error) { - r.b = append(r.b, p...) - return len(p), nil -} - -func (r *singleEchoConn) Close() error { - return nil -} - -func (r *singleEchoConn) LocalAddr() net.Addr { - return nil -} - -func (r *singleEchoConn) RemoteAddr() net.Addr { - return nil -} - -func TestSingleEchoConn(t *testing.T) { - t.Parallel() - - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return &singleEchoConn{ - b: []byte("HTTP/1.1 345 OK\r\nContent-Type: foobar\r\n\r\n"), - }, nil - }, - } - - req := Request{} - res := Response{} - - req.SetRequestURI("http://foobar/a/b") - req.Header.SetMethod("POST") - req.Header.Set("Content-Type", "text/plain") - body := bytes.NewBufferString("test") - req.SetBodyStream(body, body.Len()) - - err := c.Do(&req, &res) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if res.StatusCode() != 345 { - t.Fatalf("unexpected status code: %d. Expecting 345", res.StatusCode()) - } - expected := "POST /a/b HTTP/1.1\r\nUser-Agent: fasthttp\r\nHost: foobar\r\nContent-Type: text/plain\r\nContent-Length: 4\r\n\r\ntest" - if string(res.Body()) != expected { - t.Fatalf("unexpected body: %q. Expecting %q", res.Body(), expected) - } -} - -func TestClientHTTPSInvalidServerName(t *testing.T) { - t.Parallel() - - sHTTPS := startEchoServerTLS(t, "tcp", "127.0.0.1:") - defer sHTTPS.Stop() - - var c Client - - for i := 0; i < 10; i++ { - _, _, err := c.GetTimeout(nil, "https://"+sHTTPS.Addr(), time.Second) - if err == nil { - t.Fatalf("expecting TLS error") - } - } -} - -func TestClientHTTPSConcurrent(t *testing.T) { - t.Parallel() - - sHTTP := startEchoServer(t, "tcp", "127.0.0.1:") - defer sHTTP.Stop() - - sHTTPS := startEchoServerTLS(t, "tcp", "127.0.0.1:") - defer sHTTPS.Stop() - - c := &Client{ - TLSConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - - var wg sync.WaitGroup - for i := 0; i < 4; i++ { - wg.Add(1) - addr := "http://" + sHTTP.Addr() - if i&1 != 0 { - addr = "https://" + sHTTPS.Addr() - } - go func() { - defer wg.Done() - testClientGet(t, c, addr, 20) - testClientPost(t, c, addr, 10) - }() - } - wg.Wait() -} - -func TestClientManyServers(t *testing.T) { - t.Parallel() - - var addrs []string - for i := 0; i < 10; i++ { - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - addrs = append(addrs, s.Addr()) - } - - var wg sync.WaitGroup - for i := 0; i < 4; i++ { - wg.Add(1) - addr := "http://" + addrs[i] - go func() { - defer wg.Done() - testClientGet(t, &defaultClient, addr, 20) - testClientPost(t, &defaultClient, addr, 10) - }() - } - wg.Wait() -} - -func TestClientGet(t *testing.T) { - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - testClientGet(t, &defaultClient, "http://"+s.Addr(), 100) -} - -func TestClientPost(t *testing.T) { - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - testClientPost(t, &defaultClient, "http://"+s.Addr(), 100) -} - -func TestClientConcurrent(t *testing.T) { - t.Parallel() - - s := startEchoServer(t, "tcp", "127.0.0.1:") - defer s.Stop() - - addr := "http://" + s.Addr() - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - testClientGet(t, &defaultClient, addr, 30) - testClientPost(t, &defaultClient, addr, 10) - }() - } - wg.Wait() -} - -func skipIfNotUnix(tb testing.TB) { - switch runtime.GOOS { - case "android", "nacl", "plan9", "windows": - tb.Skipf("%s does not support unix sockets", runtime.GOOS) - } - if runtime.GOOS == "darwin" && (runtime.GOARCH == "arm" || runtime.GOARCH == "arm64") { - tb.Skip("iOS does not support unix, unixgram") - } -} - -func TestHostClientGet(t *testing.T) { - t.Parallel() - - skipIfNotUnix(t) - addr := "TestHostClientGet.unix" - s := startEchoServer(t, "unix", addr) - defer s.Stop() - c := createEchoClient(t, "unix", addr) - - testHostClientGet(t, c, 100) -} - -func TestHostClientPost(t *testing.T) { - t.Parallel() - - skipIfNotUnix(t) - addr := "./TestHostClientPost.unix" - s := startEchoServer(t, "unix", addr) - defer s.Stop() - c := createEchoClient(t, "unix", addr) - - testHostClientPost(t, c, 100) -} - -func TestHostClientConcurrent(t *testing.T) { - t.Parallel() - - skipIfNotUnix(t) - addr := "./TestHostClientConcurrent.unix" - s := startEchoServer(t, "unix", addr) - defer s.Stop() - c := createEchoClient(t, "unix", addr) - - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() - testHostClientGet(t, c, 30) - testHostClientPost(t, c, 10) - }() - } - wg.Wait() -} - -func testClientGet(t *testing.T, c clientGetter, addr string, n int) { - var buf []byte - for i := 0; i < n; i++ { - uri := fmt.Sprintf("%s/foo/%d?bar=baz", addr, i) - statusCode, body, err := c.Get(buf, uri) - buf = body - if err != nil { - t.Errorf("unexpected error when doing http request: %v", err) - } - if statusCode != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d", statusCode, StatusOK) - } - resultURI := string(body) - if resultURI != uri { - t.Errorf("unexpected uri %q. Expecting %q", resultURI, uri) - } - } -} - -func testClientDoTimeoutSuccess(t *testing.T, c *Client, addr string, n int) { - var req Request - var resp Response - - for i := 0; i < n; i++ { - uri := fmt.Sprintf("%s/foo/%d?bar=baz", addr, i) - req.SetRequestURI(uri) - if err := c.DoTimeout(&req, &resp, time.Second); err != nil { - t.Errorf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - resultURI := string(resp.Body()) - if strings.HasPrefix(uri, "https") { - resultURI = uri[:5] + resultURI[4:] - } - if resultURI != uri { - t.Errorf("unexpected uri %q. Expecting %q", resultURI, uri) - } - } -} - -func testClientGetTimeoutSuccess(t *testing.T, c *Client, addr string, n int) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - var buf []byte - for i := 0; i < n; i++ { - uri := fmt.Sprintf("%s/foo/%d?bar=baz", addr, i) - statusCode, body, err := c.GetTimeout(buf, uri, time.Second) - buf = body - if err != nil { - t.Errorf("unexpected error when doing http request: %v", err) - } - if statusCode != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d", statusCode, StatusOK) - } - resultURI := string(body) - if strings.HasPrefix(uri, "https") { - resultURI = uri[:5] + resultURI[4:] - } - if resultURI != uri { - t.Errorf("unexpected uri %q. Expecting %q", resultURI, uri) - } - } -} - -func testClientPost(t *testing.T, c clientPoster, addr string, n int) { - var buf []byte - var args Args - for i := 0; i < n; i++ { - uri := fmt.Sprintf("%s/foo/%d?bar=baz", addr, i) - args.Set("xx", fmt.Sprintf("yy%d", i)) - args.Set("zzz", fmt.Sprintf("qwe_%d", i)) - argsS := args.String() - statusCode, body, err := c.Post(buf, uri, &args) - buf = body - if err != nil { - t.Errorf("unexpected error when doing http request: %v", err) - } - if statusCode != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d", statusCode, StatusOK) - } - s := string(body) - if s != argsS { - t.Errorf("unexpected response %q. Expecting %q", s, argsS) - } - } -} - -func testHostClientGet(t *testing.T, c *HostClient, n int) { - testClientGet(t, c, "http://google.com", n) -} - -func testHostClientPost(t *testing.T, c *HostClient, n int) { - testClientPost(t, c, "http://post-host.com", n) -} - -type clientPoster interface { - Post(dst []byte, uri string, postArgs *Args) (int, []byte, error) -} - -type clientGetter interface { - Get(dst []byte, uri string) (int, []byte, error) -} - -func createEchoClient(t *testing.T, network, addr string) *HostClient { - return &HostClient{ - Addr: addr, - Dial: func(addr string) (net.Conn, error) { - return net.Dial(network, addr) - }, - } -} - -type testEchoServer struct { - s *Server - ln net.Listener - ch chan struct{} - t *testing.T -} - -func (s *testEchoServer) Stop() { - s.ln.Close() - select { - case <-s.ch: - case <-time.After(time.Second): - s.t.Fatalf("timeout when waiting for server close") - } -} - -func (s *testEchoServer) Addr() string { - return s.ln.Addr().String() -} - -func startEchoServerTLS(t *testing.T, network, addr string) *testEchoServer { - return startEchoServerExt(t, network, addr, true) -} - -func startEchoServer(t *testing.T, network, addr string) *testEchoServer { - return startEchoServerExt(t, network, addr, false) -} - -func startEchoServerExt(t *testing.T, network, addr string, isTLS bool) *testEchoServer { - if network == "unix" { - os.Remove(addr) - } - var ln net.Listener - var err error - if isTLS { - certData, keyData, kerr := GenerateTestCertificate("localhost") - if kerr != nil { - t.Fatal(kerr) - } - - cert, kerr := tls.X509KeyPair(certData, keyData) - if kerr != nil { - t.Fatal(kerr) - } - - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{cert}, - } - ln, err = tls.Listen(network, addr, tlsConfig) - } else { - ln, err = net.Listen(network, addr) - } - if err != nil { - t.Fatalf("cannot listen %q: %v", addr, err) - } - - s := &Server{ - Handler: func(ctx *RequestCtx) { - if ctx.IsGet() { - ctx.Success("text/plain", ctx.PhantomURI().FullURI()) - } else if ctx.IsPost() { - ctx.PostArgs().WriteTo(ctx) //nolint:errcheck - } - }, - Logger: &testLogger{}, // Ignore log output. - } - ch := make(chan struct{}) - go func() { - err := s.Serve(ln) - if err != nil { - t.Errorf("unexpected error returned from Serve(): %v", err) - } - close(ch) - }() - return &testEchoServer{ - s: s, - ln: ln, - ch: ch, - t: t, - } -} - -func TestClientTLSHandshakeTimeout(t *testing.T) { - t.Parallel() - - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - - addr := listener.Addr().String() - defer listener.Close() - - complete := make(chan bool) - defer close(complete) - - go func() { - conn, err := listener.Accept() - if err != nil { - t.Error(err) - return - } - <-complete - conn.Close() - }() - - client := Client{ - WriteTimeout: 100 * time.Millisecond, - ReadTimeout: 100 * time.Millisecond, - } - - _, _, err = client.Get(nil, "https://"+addr) - if err == nil { - t.Fatal("tlsClientHandshake completed successfully") - } - - if err != ErrTLSHandshakeTimeout { - t.Errorf("resulting error not a timeout: %v\nType %T: %#v", err, err, err) - } -} - -func TestHostClientMaxConnWaitTimeoutSuccess(t *testing.T) { - t.Parallel() - - var ( - emptyBodyCount uint8 - ln = fasthttputil.NewInmemoryListener() - wg sync.WaitGroup - ) - - s := &Server{ - Handler: func(ctx *RequestCtx) { - if len(ctx.PostBody()) == 0 { - emptyBodyCount++ - } - time.Sleep(5 * time.Millisecond) - ctx.WriteString("foo") //nolint:errcheck - }, - } - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &HostClient{ - Addr: "foobar", - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - MaxConns: 1, - MaxConnWaitTimeout: time.Second * 2, - } - - for i := 0; i < 5; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - req := defaultHTTPPool.AcquireRequest() - req.SetRequestURI("http://foobar/baz") - req.Header.SetMethod(MethodPost) - req.SetBodyString("bar") - resp := defaultHTTPPool.AcquireResponse() - - if err := c.Do(req, resp); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code %d. Expecting %d", resp.StatusCode(), StatusOK) - } - - body := resp.Body() - if string(body) != "foo" { - t.Errorf("unexpected body %q. Expecting %q", body, "abcd") - } - }() - } - wg.Wait() - - if c.connsWait.len() > 0 { - t.Errorf("connsWait has %v items remaining", c.connsWait.len()) - } - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second * 5): - t.Fatalf("timeout") - } - - if emptyBodyCount > 0 { - t.Fatalf("at least one request body was empty") - } -} - -func TestHostClientMaxConnWaitTimeoutError(t *testing.T) { - t.Parallel() - - var ( - emptyBodyCount uint8 - ln = fasthttputil.NewInmemoryListener() - wg sync.WaitGroup - ) - - s := &Server{ - Handler: func(ctx *RequestCtx) { - if len(ctx.PostBody()) == 0 { - emptyBodyCount++ - } - time.Sleep(5 * time.Millisecond) - ctx.WriteString("foo") //nolint:errcheck - }, - } - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &HostClient{ - Addr: "foobar", - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - MaxConns: 1, - MaxConnWaitTimeout: 10 * time.Millisecond, - } - - var errNoFreeConnsCount uint32 - for i := 0; i < 5; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - req := defaultHTTPPool.AcquireRequest() - req.SetRequestURI("http://foobar/baz") - req.Header.SetMethod(MethodPost) - req.SetBodyString("bar") - resp := defaultHTTPPool.AcquireResponse() - - if err := c.Do(req, resp); err != nil { - if err != ErrNoFreeConns { - t.Errorf("unexpected error: %v. Expecting %v", err, ErrNoFreeConns) - } - atomic.AddUint32(&errNoFreeConnsCount, 1) - } else { - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code %d. Expecting %d", resp.StatusCode(), StatusOK) - } - - body := resp.Body() - if string(body) != "foo" { - t.Errorf("unexpected body %q. Expecting %q", body, "abcd") - } - } - }() - } - wg.Wait() - - // Prevent a race condition with the conns cleaner that might still be running. - c.connsLock.Lock() - defer c.connsLock.Unlock() - - if c.connsWait.len() > 0 { - t.Errorf("connsWait has %v items remaining", c.connsWait.len()) - } - if errNoFreeConnsCount == 0 { - t.Errorf("unexpected errorCount: %d. Expecting > 0", errNoFreeConnsCount) - } - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - - if emptyBodyCount > 0 { - t.Fatalf("at least one request body was empty") - } -} - -func TestHostClientMaxConnWaitTimeoutWithEarlierDeadline(t *testing.T) { - t.Parallel() - - var ( - emptyBodyCount uint8 - ln = fasthttputil.NewInmemoryListener() - wg sync.WaitGroup - // make deadline reach earlier than conns wait timeout - sleep = 100 * time.Millisecond - timeout = 10 * time.Millisecond - maxConnWaitTimeout = 50 * time.Millisecond - ) - - s := &Server{ - Handler: func(ctx *RequestCtx) { - if len(ctx.PostBody()) == 0 { - emptyBodyCount++ - } - time.Sleep(sleep) - ctx.WriteString("foo") //nolint:errcheck - }, - Logger: &testLogger{}, // Don't print connection closed errors. - } - serverStopCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverStopCh) - }() - - c := &HostClient{ - Addr: "foobar", - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - MaxConns: 1, - MaxConnWaitTimeout: maxConnWaitTimeout, - } - - var errTimeoutCount uint32 - for i := 0; i < 5; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - req := defaultHTTPPool.AcquireRequest() - req.SetRequestURI("http://foobar/baz") - req.Header.SetMethod(MethodPost) - req.SetBodyString("bar") - resp := defaultHTTPPool.AcquireResponse() - - if err := c.DoDeadline(req, resp, time.Now().Add(timeout)); err != nil { - if err != ErrTimeout { - t.Errorf("unexpected error: %v. Expecting %v", err, ErrTimeout) - } - atomic.AddUint32(&errTimeoutCount, 1) - } else { - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code %d. Expecting %d", resp.StatusCode(), StatusOK) - } - - body := resp.Body() - if string(body) != "foo" { - t.Errorf("unexpected body %q. Expecting %q", body, "abcd") - } - } - }() - } - wg.Wait() - - c.connsLock.Lock() - for { - w := c.connsWait.popFront() - if w == nil { - break - } - w.mu.Lock() - if w.err != nil && w.err != ErrTimeout { - t.Errorf("unexpected error: %v. Expecting %v", w.err, ErrTimeout) - } - w.mu.Unlock() - } - c.connsLock.Unlock() - if errTimeoutCount == 0 { - t.Errorf("unexpected errTimeoutCount: %d. Expecting > 0", errTimeoutCount) - } - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-serverStopCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - - if emptyBodyCount > 0 { - t.Fatalf("at least one request body was empty") - } -} - -func TestHttpsRequestWithoutParsedURL(t *testing.T) { - t.Parallel() - - client := HostClient{ - IsTLS: true, - Transport: func(r1 *Request, r2 *Response) error { - return nil - }, - } - - req := &Request{} - - req.SetRequestURI("https://foo.com/bar") - - _, err := client.doNonNilReqResp(req, &Response{}) - if err != nil { - t.Fatal("https requests with IsTLS client must succeed") - } -} diff --git a/lib/fasthttp/client_timing_test.go b/lib/fasthttp/client_timing_test.go deleted file mode 100644 index f7cf78627..000000000 --- a/lib/fasthttp/client_timing_test.go +++ /dev/null @@ -1,642 +0,0 @@ -package fasthttp - -import ( - "bytes" - "fmt" - "io/ioutil" - "net" - "net/http" - "runtime" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "infini.sh/framework/lib/fasthttp/fasthttputil" -) - -type fakeClientConn struct { - net.Conn - s []byte - n int - ch chan struct{} -} - -func (c *fakeClientConn) Write(b []byte) (int, error) { - c.ch <- struct{}{} - return len(b), nil -} - -func (c *fakeClientConn) Read(b []byte) (int, error) { - if c.n == 0 { - // wait for request :) - <-c.ch - } - n := 0 - for len(b) > 0 { - if c.n == len(c.s) { - c.n = 0 - return n, nil - } - n = copy(b, c.s[c.n:]) - c.n += n - b = b[n:] - } - return n, nil -} - -func (c *fakeClientConn) Close() error { - releaseFakeServerConn(c) - return nil -} - -func (c *fakeClientConn) LocalAddr() net.Addr { - return &net.TCPAddr{ - IP: []byte{1, 2, 3, 4}, - Port: 8765, - } -} - -func (c *fakeClientConn) RemoteAddr() net.Addr { - return &net.TCPAddr{ - IP: []byte{1, 2, 3, 4}, - Port: 8765, - } -} - -func releaseFakeServerConn(c *fakeClientConn) { - c.n = 0 - fakeClientConnPool.Put(c) -} - -func acquireFakeServerConn(s []byte) *fakeClientConn { - v := fakeClientConnPool.Get() - if v == nil { - c := &fakeClientConn{ - s: s, - ch: make(chan struct{}, 1), - } - return c - } - return v.(*fakeClientConn) -} - -var fakeClientConnPool sync.Pool - -func BenchmarkClientGetTimeoutFastServer(b *testing.B) { - body := []byte("123456789099") - s := []byte(fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %d\r\n\r\n%s", len(body), body)) - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return acquireFakeServerConn(s), nil - }, - } - - nn := uint32(0) - b.RunParallel(func(pb *testing.PB) { - url := fmt.Sprintf("http://foobar%d.com/aaa/bbb", atomic.AddUint32(&nn, 1)) - var statusCode int - var bodyBuf []byte - var err error - for pb.Next() { - statusCode, bodyBuf, err = c.GetTimeout(bodyBuf[:0], url, time.Second) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - if statusCode != StatusOK { - b.Fatalf("unexpected status code: %d", statusCode) - } - if !bytes.Equal(bodyBuf, body) { - b.Fatalf("unexpected response body: %q. Expected %q", bodyBuf, body) - } - } - }) -} - -func BenchmarkClientDoFastServer(b *testing.B) { - body := []byte("012345678912") - s := []byte(fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %d\r\n\r\n%s", len(body), body)) - c := &Client{ - Dial: func(addr string) (net.Conn, error) { - return acquireFakeServerConn(s), nil - }, - MaxConnsPerHost: runtime.GOMAXPROCS(-1), - } - - nn := uint32(0) - b.RunParallel(func(pb *testing.PB) { - var req Request - var resp Response - req.Header.SetRequestURI(fmt.Sprintf("http://foobar%d.com/aaa/bbb", atomic.AddUint32(&nn, 1))) - for pb.Next() { - if err := c.Do(&req, &resp); err != nil { - b.Fatalf("unexpected error: %v", err) - } - if resp.Header.StatusCode() != StatusOK { - b.Fatalf("unexpected status code: %d", resp.Header.StatusCode()) - } - if !bytes.Equal(resp.Body(), body) { - b.Fatalf("unexpected response body: %q. Expected %q", resp.Body(), body) - } - } - }) -} - -func BenchmarkNetHTTPClientDoFastServer(b *testing.B) { - body := []byte("012345678912") - s := []byte(fmt.Sprintf("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: %d\r\n\r\n%s", len(body), body)) - c := &http.Client{ - Transport: &http.Transport{ - Dial: func(network, addr string) (net.Conn, error) { - return acquireFakeServerConn(s), nil - }, - MaxIdleConnsPerHost: runtime.GOMAXPROCS(-1), - }, - } - - nn := uint32(0) - b.RunParallel(func(pb *testing.PB) { - req, err := http.NewRequest(MethodGet, fmt.Sprintf("http://foobar%d.com/aaa/bbb", atomic.AddUint32(&nn, 1)), nil) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - for pb.Next() { - resp, err := c.Do(req) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode != http.StatusOK { - b.Fatalf("unexpected status code: %d", resp.StatusCode) - } - respBody, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - b.Fatalf("unexpected error when reading response body: %v", err) - } - if !bytes.Equal(respBody, body) { - b.Fatalf("unexpected response body: %q. Expected %q", respBody, body) - } - } - }) -} - -func fasthttpEchoHandler(ctx *RequestCtx) { - ctx.Success("text/plain", ctx.RequestURI()) -} - -func nethttpEchoHandler(w http.ResponseWriter, r *http.Request) { - w.Header().Set(HeaderContentType, "text/plain") - w.Write([]byte(r.RequestURI)) //nolint:errcheck -} - -func BenchmarkClientGetEndToEnd1TCP(b *testing.B) { - benchmarkClientGetEndToEndTCP(b, 1) -} - -func BenchmarkClientGetEndToEnd10TCP(b *testing.B) { - benchmarkClientGetEndToEndTCP(b, 10) -} - -func BenchmarkClientGetEndToEnd100TCP(b *testing.B) { - benchmarkClientGetEndToEndTCP(b, 100) -} - -func benchmarkClientGetEndToEndTCP(b *testing.B, parallelism int) { - addr := "127.0.0.1:8543" - - ln, err := net.Listen("tcp4", addr) - if err != nil { - b.Fatalf("cannot listen %q: %v", addr, err) - } - - ch := make(chan struct{}) - go func() { - if err := Serve(ln, fasthttpEchoHandler); err != nil { - b.Errorf("error when serving requests: %v", err) - } - close(ch) - }() - - c := &Client{ - MaxConnsPerHost: runtime.GOMAXPROCS(-1) * parallelism, - } - - requestURI := "/foo/bar?baz=123" - url := "http://" + addr + requestURI - b.SetParallelism(parallelism) - b.RunParallel(func(pb *testing.PB) { - var buf []byte - for pb.Next() { - statusCode, body, err := c.Get(buf, url) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - if statusCode != StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", statusCode, StatusOK) - } - if string(body) != requestURI { - b.Fatalf("unexpected response %q. Expecting %q", body, requestURI) - } - buf = body - } - }) - - ln.Close() - select { - case <-ch: - case <-time.After(time.Second): - b.Fatalf("server wasn't stopped") - } -} - -func BenchmarkNetHTTPClientGetEndToEnd1TCP(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndTCP(b, 1) -} - -func BenchmarkNetHTTPClientGetEndToEnd10TCP(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndTCP(b, 10) -} - -func BenchmarkNetHTTPClientGetEndToEnd100TCP(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndTCP(b, 100) -} - -func benchmarkNetHTTPClientGetEndToEndTCP(b *testing.B, parallelism int) { - addr := "127.0.0.1:8542" - - ln, err := net.Listen("tcp4", addr) - if err != nil { - b.Fatalf("cannot listen %q: %v", addr, err) - } - - ch := make(chan struct{}) - go func() { - if err := http.Serve(ln, http.HandlerFunc(nethttpEchoHandler)); err != nil && !strings.Contains( - err.Error(), "use of closed network connection") { - b.Errorf("error when serving requests: %v", err) - } - close(ch) - }() - - c := &http.Client{ - Transport: &http.Transport{ - MaxIdleConnsPerHost: parallelism * runtime.GOMAXPROCS(-1), - }, - } - - requestURI := "/foo/bar?baz=123" - url := "http://" + addr + requestURI - b.SetParallelism(parallelism) - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - resp, err := c.Get(url) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode != http.StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode, http.StatusOK) - } - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - b.Fatalf("unexpected error when reading response body: %v", err) - } - if string(body) != requestURI { - b.Fatalf("unexpected response %q. Expecting %q", body, requestURI) - } - } - }) - - ln.Close() - select { - case <-ch: - case <-time.After(time.Second): - b.Fatalf("server wasn't stopped") - } -} - -func BenchmarkClientGetEndToEnd1Inmemory(b *testing.B) { - benchmarkClientGetEndToEndInmemory(b, 1) -} - -func BenchmarkClientGetEndToEnd10Inmemory(b *testing.B) { - benchmarkClientGetEndToEndInmemory(b, 10) -} - -func BenchmarkClientGetEndToEnd100Inmemory(b *testing.B) { - benchmarkClientGetEndToEndInmemory(b, 100) -} - -func BenchmarkClientGetEndToEnd1000Inmemory(b *testing.B) { - benchmarkClientGetEndToEndInmemory(b, 1000) -} - -func BenchmarkClientGetEndToEnd10KInmemory(b *testing.B) { - benchmarkClientGetEndToEndInmemory(b, 10000) -} - -func benchmarkClientGetEndToEndInmemory(b *testing.B, parallelism int) { - ln := fasthttputil.NewInmemoryListener() - - ch := make(chan struct{}) - go func() { - if err := Serve(ln, fasthttpEchoHandler); err != nil { - b.Errorf("error when serving requests: %v", err) - } - close(ch) - }() - - c := &Client{ - MaxConnsPerHost: runtime.GOMAXPROCS(-1) * parallelism, - Dial: func(addr string) (net.Conn, error) { return ln.Dial() }, - } - - requestURI := "/foo/bar?baz=123" - url := "http://unused.host" + requestURI - b.SetParallelism(parallelism) - b.RunParallel(func(pb *testing.PB) { - var buf []byte - for pb.Next() { - statusCode, body, err := c.Get(buf, url) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - if statusCode != StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", statusCode, StatusOK) - } - if string(body) != requestURI { - b.Fatalf("unexpected response %q. Expecting %q", body, requestURI) - } - buf = body - } - }) - - ln.Close() - select { - case <-ch: - case <-time.After(time.Second): - b.Fatalf("server wasn't stopped") - } -} - -func BenchmarkNetHTTPClientGetEndToEnd1Inmemory(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndInmemory(b, 1) -} - -func BenchmarkNetHTTPClientGetEndToEnd10Inmemory(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndInmemory(b, 10) -} - -func BenchmarkNetHTTPClientGetEndToEnd100Inmemory(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndInmemory(b, 100) -} - -func BenchmarkNetHTTPClientGetEndToEnd1000Inmemory(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndInmemory(b, 1000) -} - -func benchmarkNetHTTPClientGetEndToEndInmemory(b *testing.B, parallelism int) { - ln := fasthttputil.NewInmemoryListener() - - ch := make(chan struct{}) - go func() { - if err := http.Serve(ln, http.HandlerFunc(nethttpEchoHandler)); err != nil && !strings.Contains( - err.Error(), "use of closed network connection") { - b.Errorf("error when serving requests: %v", err) - } - close(ch) - }() - - c := &http.Client{ - Transport: &http.Transport{ - Dial: func(_, _ string) (net.Conn, error) { return ln.Dial() }, - MaxIdleConnsPerHost: parallelism * runtime.GOMAXPROCS(-1), - }, - } - - requestURI := "/foo/bar?baz=123" - url := "http://unused.host" + requestURI - b.SetParallelism(parallelism) - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - resp, err := c.Get(url) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode != http.StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode, http.StatusOK) - } - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - b.Fatalf("unexpected error when reading response body: %v", err) - } - if string(body) != requestURI { - b.Fatalf("unexpected response %q. Expecting %q", body, requestURI) - } - } - }) - - ln.Close() - select { - case <-ch: - case <-time.After(time.Second): - b.Fatalf("server wasn't stopped") - } -} - -func BenchmarkClientEndToEndBigResponse1Inmemory(b *testing.B) { - benchmarkClientEndToEndBigResponseInmemory(b, 1) -} - -func BenchmarkClientEndToEndBigResponse10Inmemory(b *testing.B) { - benchmarkClientEndToEndBigResponseInmemory(b, 10) -} - -func benchmarkClientEndToEndBigResponseInmemory(b *testing.B, parallelism int) { - bigResponse := createFixedBody(1024 * 1024) - h := func(ctx *RequestCtx) { - ctx.SetContentType("text/plain") - ctx.Write(bigResponse) //nolint:errcheck - } - - ln := fasthttputil.NewInmemoryListener() - - ch := make(chan struct{}) - go func() { - if err := Serve(ln, h); err != nil { - b.Errorf("error when serving requests: %v", err) - } - close(ch) - }() - - c := &Client{ - MaxConnsPerHost: runtime.GOMAXPROCS(-1) * parallelism, - Dial: func(addr string) (net.Conn, error) { return ln.Dial() }, - } - - requestURI := "/foo/bar?baz=123" - url := "http://unused.host" + requestURI - b.SetParallelism(parallelism) - b.RunParallel(func(pb *testing.PB) { - var req Request - req.SetRequestURI(url) - var resp Response - for pb.Next() { - if err := c.DoTimeout(&req, &resp, 5*time.Second); err != nil { - b.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - body := resp.Body() - if !bytes.Equal(bigResponse, body) { - b.Fatalf("unexpected response %q. Expecting %q", body, bigResponse) - } - } - }) - - ln.Close() - select { - case <-ch: - case <-time.After(time.Second): - b.Fatalf("server wasn't stopped") - } -} - -func BenchmarkNetHTTPClientEndToEndBigResponse1Inmemory(b *testing.B) { - benchmarkNetHTTPClientEndToEndBigResponseInmemory(b, 1) -} - -func BenchmarkNetHTTPClientEndToEndBigResponse10Inmemory(b *testing.B) { - benchmarkNetHTTPClientEndToEndBigResponseInmemory(b, 10) -} - -func benchmarkNetHTTPClientEndToEndBigResponseInmemory(b *testing.B, parallelism int) { - bigResponse := createFixedBody(1024 * 1024) - h := func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set(HeaderContentType, "text/plain") - w.Write(bigResponse) //nolint:errcheck - } - ln := fasthttputil.NewInmemoryListener() - - ch := make(chan struct{}) - go func() { - if err := http.Serve(ln, http.HandlerFunc(h)); err != nil && !strings.Contains( - err.Error(), "use of closed network connection") { - b.Errorf("error when serving requests: %v", err) - } - close(ch) - }() - - c := &http.Client{ - Transport: &http.Transport{ - Dial: func(_, _ string) (net.Conn, error) { return ln.Dial() }, - MaxIdleConnsPerHost: parallelism * runtime.GOMAXPROCS(-1), - }, - Timeout: 5 * time.Second, - } - - requestURI := "/foo/bar?baz=123" - url := "http://unused.host" + requestURI - b.SetParallelism(parallelism) - b.RunParallel(func(pb *testing.PB) { - req, err := http.NewRequest(MethodGet, url, nil) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - for pb.Next() { - resp, err := c.Do(req) - if err != nil { - b.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode != http.StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode, http.StatusOK) - } - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - b.Fatalf("unexpected error when reading response body: %v", err) - } - if !bytes.Equal(bigResponse, body) { - b.Fatalf("unexpected response %q. Expecting %q", body, bigResponse) - } - } - }) - - ln.Close() - select { - case <-ch: - case <-time.After(time.Second): - b.Fatalf("server wasn't stopped") - } -} - -func BenchmarkPipelineClient1(b *testing.B) { - benchmarkPipelineClient(b, 1) -} - -func BenchmarkPipelineClient10(b *testing.B) { - benchmarkPipelineClient(b, 10) -} - -func BenchmarkPipelineClient100(b *testing.B) { - benchmarkPipelineClient(b, 100) -} - -func BenchmarkPipelineClient1000(b *testing.B) { - benchmarkPipelineClient(b, 1000) -} - -func benchmarkPipelineClient(b *testing.B, parallelism int) { - h := func(ctx *RequestCtx) { - ctx.WriteString("foobar") //nolint:errcheck - } - ln := fasthttputil.NewInmemoryListener() - - ch := make(chan struct{}) - go func() { - if err := Serve(ln, h); err != nil { - b.Errorf("error when serving requests: %v", err) - } - close(ch) - }() - - maxConns := runtime.GOMAXPROCS(-1) - c := &PipelineClient{ - Dial: func(addr string) (net.Conn, error) { return ln.Dial() }, - ReadBufferSize: 1024 * 1024, - WriteBufferSize: 1024 * 1024, - MaxConns: maxConns, - MaxPendingRequests: parallelism * maxConns, - } - - requestURI := "/foo/bar?baz=123" - url := "http://unused.host" + requestURI - b.SetParallelism(parallelism) - b.RunParallel(func(pb *testing.PB) { - var req Request - req.SetRequestURI(url) - var resp Response - for pb.Next() { - if err := c.Do(&req, &resp); err != nil { - b.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - body := resp.Body() - if string(body) != "foobar" { - b.Fatalf("unexpected response %q. Expecting %q", body, "foobar") - } - } - }) - - ln.Close() - select { - case <-ch: - case <-time.After(time.Second): - b.Fatalf("server wasn't stopped") - } -} diff --git a/lib/fasthttp/client_timing_wait_test.go b/lib/fasthttp/client_timing_wait_test.go deleted file mode 100644 index c8b3b11a3..000000000 --- a/lib/fasthttp/client_timing_wait_test.go +++ /dev/null @@ -1,167 +0,0 @@ -//go:build go1.11 -// +build go1.11 - -package fasthttp - -import ( - "io/ioutil" - "net" - "net/http" - "strings" - "testing" - "time" - - "infini.sh/framework/lib/fasthttp/fasthttputil" -) - -func newFasthttpSleepEchoHandler(sleep time.Duration) RequestHandler { - return func(ctx *RequestCtx) { - time.Sleep(sleep) - ctx.Success("text/plain", ctx.RequestURI()) - } -} - -func BenchmarkClientGetEndToEndWaitConn1Inmemory(b *testing.B) { - benchmarkClientGetEndToEndWaitConnInmemory(b, 1) -} - -func BenchmarkClientGetEndToEndWaitConn10Inmemory(b *testing.B) { - benchmarkClientGetEndToEndWaitConnInmemory(b, 10) -} - -func BenchmarkClientGetEndToEndWaitConn100Inmemory(b *testing.B) { - benchmarkClientGetEndToEndWaitConnInmemory(b, 100) -} - -func BenchmarkClientGetEndToEndWaitConn1000Inmemory(b *testing.B) { - benchmarkClientGetEndToEndWaitConnInmemory(b, 1000) -} - -func benchmarkClientGetEndToEndWaitConnInmemory(b *testing.B, parallelism int) { - ln := fasthttputil.NewInmemoryListener() - - ch := make(chan struct{}) - sleepDuration := 50 * time.Millisecond - go func() { - - if err := Serve(ln, newFasthttpSleepEchoHandler(sleepDuration)); err != nil { - b.Errorf("error when serving requests: %v", err) - } - close(ch) - }() - - c := &Client{ - MaxConnsPerHost: 1, - Dial: func(addr string) (net.Conn, error) { return ln.Dial() }, - MaxConnWaitTimeout: 5 * time.Second, - } - - requestURI := "/foo/bar?baz=123&sleep=10ms" - url := "http://unused.host" + requestURI - b.SetParallelism(parallelism) - b.RunParallel(func(pb *testing.PB) { - var buf []byte - for pb.Next() { - statusCode, body, err := c.Get(buf, url) - if err != nil { - if err != ErrNoFreeConns { - b.Fatalf("unexpected error: %v", err) - } - } else { - if statusCode != StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", statusCode, StatusOK) - } - if string(body) != requestURI { - b.Fatalf("unexpected response %q. Expecting %q", body, requestURI) - } - } - buf = body - } - }) - - ln.Close() - select { - case <-ch: - case <-time.After(time.Second): - b.Fatalf("server wasn't stopped") - } -} - -func newNethttpSleepEchoHandler(sleep time.Duration) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - time.Sleep(sleep) - w.Header().Set(HeaderContentType, "text/plain") - w.Write([]byte(r.RequestURI)) //nolint:errcheck - } -} - -func BenchmarkNetHTTPClientGetEndToEndWaitConn1Inmemory(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndWaitConnInmemory(b, 1) -} - -func BenchmarkNetHTTPClientGetEndToEndWaitConn10Inmemory(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndWaitConnInmemory(b, 10) -} - -func BenchmarkNetHTTPClientGetEndToEndWaitConn100Inmemory(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndWaitConnInmemory(b, 100) -} - -func BenchmarkNetHTTPClientGetEndToEndWaitConn1000Inmemory(b *testing.B) { - benchmarkNetHTTPClientGetEndToEndWaitConnInmemory(b, 1000) -} - -func benchmarkNetHTTPClientGetEndToEndWaitConnInmemory(b *testing.B, parallelism int) { - ln := fasthttputil.NewInmemoryListener() - - ch := make(chan struct{}) - sleep := 50 * time.Millisecond - go func() { - if err := http.Serve(ln, newNethttpSleepEchoHandler(sleep)); err != nil && !strings.Contains( - err.Error(), "use of closed network connection") { - b.Errorf("error when serving requests: %v", err) - } - close(ch) - }() - - c := &http.Client{ - Transport: &http.Transport{ - Dial: func(_, _ string) (net.Conn, error) { return ln.Dial() }, - MaxConnsPerHost: 1, - }, - Timeout: 5 * time.Second, - } - - requestURI := "/foo/bar?baz=123" - url := "http://unused.host" + requestURI - b.SetParallelism(parallelism) - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - resp, err := c.Get(url) - if err != nil { - if netErr, ok := err.(net.Error); !ok || !netErr.Timeout() { - b.Fatalf("unexpected error: %v", err) - } - } else { - if resp.StatusCode != http.StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode, http.StatusOK) - } - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - b.Fatalf("unexpected error when reading response body: %v", err) - } - if string(body) != requestURI { - b.Fatalf("unexpected response %q. Expecting %q", body, requestURI) - } - } - } - }) - - ln.Close() - select { - case <-ch: - case <-time.After(time.Second): - b.Fatalf("server wasn't stopped") - } -} diff --git a/lib/fasthttp/client_unix_test.go b/lib/fasthttp/client_unix_test.go deleted file mode 100644 index 517752635..000000000 --- a/lib/fasthttp/client_unix_test.go +++ /dev/null @@ -1,136 +0,0 @@ -//go:build !windows -// +build !windows - -package fasthttp - -import ( - "io" - "io/ioutil" - "net" - "net/http" - "strings" - "testing" -) - -// See issue #1232 -func TestRstConnResponseWhileSending(t *testing.T) { - const expectedStatus = http.StatusTeapot - const payload = "payload" - - srv, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer srv.Close() - - go func() { - for { - conn, err := srv.Accept() - if err != nil { - return - } - - // Read at least one byte of the header - // Otherwise we would have an unsolicited response - _, err = ioutil.ReadAll(io.LimitReader(conn, 1)) - if err != nil { - t.Error(err) - } - - // Respond - _, err = conn.Write([]byte("HTTP/1.1 418 Teapot\r\n\r\n")) - if err != nil { - t.Error(err) - } - - // Forcefully close connection - err = conn.(*net.TCPConn).SetLinger(0) - if err != nil { - t.Error(err) - } - conn.Close() - } - }() - - svrUrl := "http://" + srv.Addr().String() - client := HostClient{Addr: srv.Addr().String()} - - for i := 0; i < 100; i++ { - req := defaultHTTPPool.AcquireRequest() - defer defaultHTTPPool.ReleaseRequest(req) - resp := defaultHTTPPool.AcquireResponse() - defer defaultHTTPPool.ReleaseResponse(resp) - - req.Header.SetMethod("POST") - req.SetBodyStream(strings.NewReader(payload), len(payload)) - req.SetRequestURI(svrUrl) - - err = client.Do(req, resp) - if err != nil { - t.Fatal(err) - } - if expectedStatus != resp.StatusCode() { - t.Fatalf("Expected %d status code, but got %d", expectedStatus, resp.StatusCode()) - } - } -} - -// See issue #1232 -func TestRstConnClosedWithoutResponse(t *testing.T) { - const payload = "payload" - - srv, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer srv.Close() - - go func() { - for { - conn, err := srv.Accept() - if err != nil { - return - } - - // Read at least one byte of the header - // Otherwise we would have an unsolicited response - _, err = ioutil.ReadAll(io.LimitReader(conn, 1)) - if err != nil { - t.Error(err) - } - - // Respond with incomplete header - _, err = conn.Write([]byte("Http")) - if err != nil { - t.Error(err) - } - - // Forcefully close connection - err = conn.(*net.TCPConn).SetLinger(0) - if err != nil { - t.Error(err) - } - conn.Close() - } - }() - - svrUrl := "http://" + srv.Addr().String() - client := HostClient{Addr: srv.Addr().String()} - - for i := 0; i < 100; i++ { - req := defaultHTTPPool.AcquireRequest() - defer defaultHTTPPool.ReleaseRequest(req) - resp := defaultHTTPPool.AcquireResponse() - defer defaultHTTPPool.ReleaseResponse(resp) - - req.Header.SetMethod("POST") - req.SetBodyStream(strings.NewReader(payload), len(payload)) - req.SetRequestURI(svrUrl) - - err = client.Do(req, resp) - - if !isConnectionReset(err) { - t.Fatal("Expected connection reset error") - } - } -} diff --git a/lib/fasthttp/coarseTime_test.go b/lib/fasthttp/coarseTime_test.go deleted file mode 100644 index b2f2334ee..000000000 --- a/lib/fasthttp/coarseTime_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package fasthttp - -import ( - "sync/atomic" - "testing" - "time" -) - -func BenchmarkCoarseTimeNow(b *testing.B) { - var zeroTimeCount uint64 - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - t := CoarseTimeNow() - if t.IsZero() { - atomic.AddUint64(&zeroTimeCount, 1) - } - } - }) - if zeroTimeCount > 0 { - b.Fatalf("zeroTimeCount must be zero") - } -} - -func BenchmarkTimeNow(b *testing.B) { - var zeroTimeCount uint64 - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - t := time.Now() - if t.IsZero() { - atomic.AddUint64(&zeroTimeCount, 1) - } - } - }) - if zeroTimeCount > 0 { - b.Fatalf("zeroTimeCount must be zero") - } -} diff --git a/lib/fasthttp/compress_test.go b/lib/fasthttp/compress_test.go deleted file mode 100644 index 111c93c61..000000000 --- a/lib/fasthttp/compress_test.go +++ /dev/null @@ -1,232 +0,0 @@ -package fasthttp - -import ( - "bytes" - "fmt" - "io/ioutil" - "testing" - "time" -) - -var compressTestcases = func() []string { - a := []string{ - "", - "foobar", - "выфаодлодл одлфываыв sd2 k34", - } - bigS := createFixedBody(1e4) - a = append(a, string(bigS)) - return a -}() - -func TestGzipBytesSerial(t *testing.T) { - t.Parallel() - - if err := testGzipBytes(); err != nil { - t.Fatal(err) - } -} - -func TestGzipBytesConcurrent(t *testing.T) { - t.Parallel() - - if err := testConcurrent(10, testGzipBytes); err != nil { - t.Fatal(err) - } -} - -func TestDeflateBytesSerial(t *testing.T) { - t.Parallel() - - if err := testDeflateBytes(); err != nil { - t.Fatal(err) - } -} - -func TestDeflateBytesConcurrent(t *testing.T) { - t.Parallel() - - if err := testConcurrent(10, testDeflateBytes); err != nil { - t.Fatal(err) - } -} - -func testGzipBytes() error { - for _, s := range compressTestcases { - if err := testGzipBytesSingleCase(s); err != nil { - return err - } - } - return nil -} - -func testDeflateBytes() error { - for _, s := range compressTestcases { - if err := testDeflateBytesSingleCase(s); err != nil { - return err - } - } - return nil -} - -func testGzipBytesSingleCase(s string) error { - prefix := []byte("foobar") - gzippedS := AppendGzipBytes(prefix, []byte(s)) - if !bytes.Equal(gzippedS[:len(prefix)], prefix) { - return fmt.Errorf("unexpected prefix when compressing %q: %q. Expecting %q", s, gzippedS[:len(prefix)], prefix) - } - - gunzippedS, err := AppendGunzipBytes(prefix, gzippedS[len(prefix):]) - if err != nil { - return fmt.Errorf("unexpected error when uncompressing %q: %w", s, err) - } - if !bytes.Equal(gunzippedS[:len(prefix)], prefix) { - return fmt.Errorf("unexpected prefix when uncompressing %q: %q. Expecting %q", s, gunzippedS[:len(prefix)], prefix) - } - gunzippedS = gunzippedS[len(prefix):] - if string(gunzippedS) != s { - return fmt.Errorf("unexpected uncompressed string %q. Expecting %q", gunzippedS, s) - } - return nil -} - -func testDeflateBytesSingleCase(s string) error { - prefix := []byte("foobar") - deflatedS := AppendDeflateBytes(prefix, []byte(s)) - if !bytes.Equal(deflatedS[:len(prefix)], prefix) { - return fmt.Errorf("unexpected prefix when compressing %q: %q. Expecting %q", s, deflatedS[:len(prefix)], prefix) - } - - inflatedS, err := AppendInflateBytes(prefix, deflatedS[len(prefix):]) - if err != nil { - return fmt.Errorf("unexpected error when uncompressing %q: %w", s, err) - } - if !bytes.Equal(inflatedS[:len(prefix)], prefix) { - return fmt.Errorf("unexpected prefix when uncompressing %q: %q. Expecting %q", s, inflatedS[:len(prefix)], prefix) - } - inflatedS = inflatedS[len(prefix):] - if string(inflatedS) != s { - return fmt.Errorf("unexpected uncompressed string %q. Expecting %q", inflatedS, s) - } - return nil -} - -func TestGzipCompressSerial(t *testing.T) { - t.Parallel() - - if err := testGzipCompress(); err != nil { - t.Fatal(err) - } -} - -func TestGzipCompressConcurrent(t *testing.T) { - t.Parallel() - - if err := testConcurrent(10, testGzipCompress); err != nil { - t.Fatal(err) - } -} - -func TestFlateCompressSerial(t *testing.T) { - t.Parallel() - - if err := testFlateCompress(); err != nil { - t.Fatal(err) - } -} - -func TestFlateCompressConcurrent(t *testing.T) { - t.Parallel() - - if err := testConcurrent(10, testFlateCompress); err != nil { - t.Fatal(err) - } -} - -func testGzipCompress() error { - for _, s := range compressTestcases { - if err := testGzipCompressSingleCase(s); err != nil { - return err - } - } - return nil -} - -func testFlateCompress() error { - for _, s := range compressTestcases { - if err := testFlateCompressSingleCase(s); err != nil { - return err - } - } - return nil -} - -func testGzipCompressSingleCase(s string) error { - var buf bytes.Buffer - zw := acquireStacklessGzipWriter(&buf, CompressDefaultCompression) - if _, err := zw.Write([]byte(s)); err != nil { - return fmt.Errorf("unexpected error: %w. s=%q", err, s) - } - releaseStacklessGzipWriter(zw, CompressDefaultCompression) - - zr, err := acquireGzipReader(&buf) - if err != nil { - return fmt.Errorf("unexpected error: %w. s=%q", err, s) - } - body, err := ioutil.ReadAll(zr) - if err != nil { - return fmt.Errorf("unexpected error: %w. s=%q", err, s) - } - if string(body) != s { - return fmt.Errorf("unexpected string after decompression: %q. Expecting %q", body, s) - } - releaseGzipReader(zr) - return nil -} - -func testFlateCompressSingleCase(s string) error { - var buf bytes.Buffer - zw := acquireStacklessDeflateWriter(&buf, CompressDefaultCompression) - if _, err := zw.Write([]byte(s)); err != nil { - return fmt.Errorf("unexpected error: %w. s=%q", err, s) - } - releaseStacklessDeflateWriter(zw, CompressDefaultCompression) - - zr, err := acquireFlateReader(&buf) - if err != nil { - return fmt.Errorf("unexpected error: %w. s=%q", err, s) - } - body, err := ioutil.ReadAll(zr) - if err != nil { - return fmt.Errorf("unexpected error: %w. s=%q", err, s) - } - if string(body) != s { - return fmt.Errorf("unexpected string after decompression: %q. Expecting %q", body, s) - } - releaseFlateReader(zr) - return nil -} - -func testConcurrent(concurrency int, f func() error) error { - ch := make(chan error, concurrency) - for i := 0; i < concurrency; i++ { - go func(idx int) { - err := f() - if err != nil { - ch <- fmt.Errorf("error in goroutine %d: %w", idx, err) - } - ch <- nil - }(i) - } - for i := 0; i < concurrency; i++ { - select { - case err := <-ch: - if err != nil { - return err - } - case <-time.After(time.Second): - return fmt.Errorf("timeout") - } - } - return nil -} diff --git a/lib/fasthttp/cookie_test.go b/lib/fasthttp/cookie_test.go deleted file mode 100644 index b4b81ac9a..000000000 --- a/lib/fasthttp/cookie_test.go +++ /dev/null @@ -1,414 +0,0 @@ -package fasthttp - -import ( - "strings" - "testing" - "time" -) - -func TestCookiePanic(t *testing.T) { - t.Parallel() - - var c Cookie - if err := c.Parse(";SAMeSITe="); err != nil { - t.Error(err) - } -} - -func TestCookieValueWithEqualAndSpaceChars(t *testing.T) { - t.Parallel() - - testCookieValueWithEqualAndSpaceChars(t, "sth1", "/", "MTQ2NjU5NTcwN3xfUVduVXk4aG9jSmZaNzNEb1dGa1VjekY1bG9vMmxSWlJBZUN2Q1ZtZVFNMTk2YU9YaWtCVmY1eDRWZXd3M3Q5RTJRZnZMbk5mWklSSFZJcVlXTDhiSFFHWWdpdFVLd1hwbXR2UUN4QlJ1N3BITFpkS3Y4PXzDvPNn6JVDBFB2wYVYPHdkdlZBm6n1_0QB3_GWwE40Tg ==") - testCookieValueWithEqualAndSpaceChars(t, "sth2", "/", "123") - testCookieValueWithEqualAndSpaceChars(t, "sth3", "/", "123 == 1") -} - -func testCookieValueWithEqualAndSpaceChars(t *testing.T, expectedName, expectedPath, expectedValue string) { - var c Cookie - c.SetKey(expectedName) - c.SetPath(expectedPath) - c.SetValue(expectedValue) - - s := c.String() - - var c1 Cookie - if err := c1.Parse(s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - name := c1.Key() - if string(name) != expectedName { - t.Fatalf("unexpected name %q. Expecting %q", name, expectedName) - } - path := c1.Path() - if string(path) != expectedPath { - t.Fatalf("unexpected path %q. Expecting %q", path, expectedPath) - } - value := c1.Value() - if string(value) != expectedValue { - t.Fatalf("unexpected value %q. Expecting %q", value, expectedValue) - } -} - -func TestCookieSecureHttpOnly(t *testing.T) { - t.Parallel() - - var c Cookie - - if err := c.Parse("foo=bar; HttpOnly; secure"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !c.Secure() { - t.Fatalf("secure must be set") - } - if !c.HTTPOnly() { - t.Fatalf("HttpOnly must be set") - } - s := c.String() - if !strings.Contains(s, "; secure") { - t.Fatalf("missing secure flag in cookie %q", s) - } - if !strings.Contains(s, "; HttpOnly") { - t.Fatalf("missing HttpOnly flag in cookie %q", s) - } -} - -func TestCookieSecure(t *testing.T) { - t.Parallel() - - var c Cookie - - if err := c.Parse("foo=bar; secure"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !c.Secure() { - t.Fatalf("secure must be set") - } - s := c.String() - if !strings.Contains(s, "; secure") { - t.Fatalf("missing secure flag in cookie %q", s) - } - - if err := c.Parse("foo=bar"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if c.Secure() { - t.Fatalf("Unexpected secure flag set") - } - s = c.String() - if strings.Contains(s, "secure") { - t.Fatalf("unexpected secure flag in cookie %q", s) - } -} - -func TestCookieSameSite(t *testing.T) { - t.Parallel() - - var c Cookie - - if err := c.Parse("foo=bar; samesite"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if c.SameSite() != CookieSameSiteDefaultMode { - t.Fatalf("SameSite must be set") - } - s := c.String() - if !strings.Contains(s, "; SameSite") { - t.Fatalf("missing SameSite flag in cookie %q", s) - } - - if err := c.Parse("foo=bar; samesite=lax"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if c.SameSite() != CookieSameSiteLaxMode { - t.Fatalf("SameSite Lax Mode must be set") - } - s = c.String() - if !strings.Contains(s, "; SameSite=Lax") { - t.Fatalf("missing SameSite flag in cookie %q", s) - } - - if err := c.Parse("foo=bar; samesite=strict"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if c.SameSite() != CookieSameSiteStrictMode { - t.Fatalf("SameSite Strict Mode must be set") - } - s = c.String() - if !strings.Contains(s, "; SameSite=Strict") { - t.Fatalf("missing SameSite flag in cookie %q", s) - } - - if err := c.Parse("foo=bar; samesite=none"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if c.SameSite() != CookieSameSiteNoneMode { - t.Fatalf("SameSite None Mode must be set") - } - s = c.String() - if !strings.Contains(s, "; SameSite=None") { - t.Fatalf("missing SameSite flag in cookie %q", s) - } - - if err := c.Parse("foo=bar"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - c.SetSameSite(CookieSameSiteNoneMode) - s = c.String() - if !strings.Contains(s, "; SameSite=None") { - t.Fatalf("missing SameSite flag in cookie %q", s) - } - if !strings.Contains(s, "; secure") { - t.Fatalf("missing Secure flag in cookie %q", s) - } - - if err := c.Parse("foo=bar"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if c.SameSite() != CookieSameSiteDisabled { - t.Fatalf("Unexpected SameSite flag set") - } - s = c.String() - if strings.Contains(s, "SameSite") { - t.Fatalf("unexpected SameSite flag in cookie %q", s) - } -} - -func TestCookieMaxAge(t *testing.T) { - t.Parallel() - - var c Cookie - - maxAge := 100 - if err := c.Parse("foo=bar; max-age=100"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if maxAge != c.MaxAge() { - t.Fatalf("max-age must be set") - } - s := c.String() - if !strings.Contains(s, "; max-age=100") { - t.Fatalf("missing max-age flag in cookie %q", s) - } - - if err := c.Parse("foo=bar; expires=Tue, 10 Nov 2009 23:00:00 GMT; max-age=100;"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if maxAge != c.MaxAge() { - t.Fatalf("max-age ignored") - } - s = c.String() - if s != "foo=bar; max-age=100" { - t.Fatalf("missing max-age in cookie %q", s) - } - - expires := time.Unix(100, 0) - c.SetExpire(expires) - s = c.String() - if s != "foo=bar; max-age=100" { - t.Fatalf("expires should be ignored due to max-age: %q", s) - } - - c.SetMaxAge(0) - s = c.String() - if s != "foo=bar; expires=Thu, 01 Jan 1970 00:01:40 GMT" { - t.Fatalf("missing expires %q", s) - } -} - -func TestCookieHttpOnly(t *testing.T) { - t.Parallel() - - var c Cookie - - if err := c.Parse("foo=bar; HttpOnly"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !c.HTTPOnly() { - t.Fatalf("HTTPOnly must be set") - } - s := c.String() - if !strings.Contains(s, "; HttpOnly") { - t.Fatalf("missing HttpOnly flag in cookie %q", s) - } - - if err := c.Parse("foo=bar"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if c.HTTPOnly() { - t.Fatalf("Unexpected HTTPOnly flag set") - } - s = c.String() - if strings.Contains(s, "HttpOnly") { - t.Fatalf("unexpected HttpOnly flag in cookie %q", s) - } -} - -func TestCookieAcquireReleaseSequential(t *testing.T) { - t.Parallel() - - testCookieAcquireRelease(t) -} - -func TestCookieAcquireReleaseConcurrent(t *testing.T) { - t.Parallel() - - ch := make(chan struct{}, 10) - for i := 0; i < 10; i++ { - go func() { - testCookieAcquireRelease(t) - ch <- struct{}{} - }() - } - for i := 0; i < 10; i++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } -} - -func testCookieAcquireRelease(t *testing.T) { - c := AcquireCookie() - - key := "foo" - c.SetKey(key) - - value := "bar" - c.SetValue(value) - - domain := "foo.bar.com" - c.SetDomain(domain) - - path := "/foi/bar/aaa" - c.SetPath(path) - - s := c.String() - c.Reset() - if err := c.Parse(s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if string(c.Key()) != key { - t.Fatalf("unexpected cookie name %q. Expecting %q", c.Key(), key) - } - if string(c.Value()) != value { - t.Fatalf("unexpected cookie value %q. Expecting %q", c.Value(), value) - } - if string(c.Domain()) != domain { - t.Fatalf("unexpected domain %q. Expecting %q", c.Domain(), domain) - } - if string(c.Path()) != path { - t.Fatalf("unexpected path %q. Expecting %q", c.Path(), path) - } - - ReleaseCookie(c) -} - -func TestCookieParse(t *testing.T) { - t.Parallel() - - testCookieParse(t, "foo", "foo") - testCookieParse(t, "foo=bar", "foo=bar") - testCookieParse(t, "foo=", "foo=") - testCookieParse(t, `foo="bar"`, "foo=bar") - testCookieParse(t, `"foo"=bar`, `"foo"=bar`) - testCookieParse(t, "foo=bar; Domain=aaa.com; PATH=/foo/bar", "foo=bar; domain=aaa.com; path=/foo/bar") - testCookieParse(t, "foo=bar; max-age= 101 ; expires= Tue, 10 Nov 2009 23:00:00 GMT", "foo=bar; max-age=101") - testCookieParse(t, " xxx = yyy ; path=/a/b;;;domain=foobar.com ; expires= Tue, 10 Nov 2009 23:00:00 GMT ; ;;", - "xxx=yyy; expires=Tue, 10 Nov 2009 23:00:00 GMT; domain=foobar.com; path=/a/b") -} - -func testCookieParse(t *testing.T, s, expectedS string) { - var c Cookie - if err := c.Parse(s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - result := string(c.Cookie()) - if result != expectedS { - t.Fatalf("unexpected cookies %q. Expecting %q. Original %q", result, expectedS, s) - } -} - -func TestCookieAppendBytes(t *testing.T) { - t.Parallel() - - c := &Cookie{} - - testCookieAppendBytes(t, c, "", "bar", "bar") - testCookieAppendBytes(t, c, "foo", "", "foo=") - testCookieAppendBytes(t, c, "ффф", "12 лодлы", "ффф=12 лодлы") - - c.SetDomain("foobar.com") - testCookieAppendBytes(t, c, "a", "b", "a=b; domain=foobar.com") - - c.SetPath("/a/b") - testCookieAppendBytes(t, c, "aa", "bb", "aa=bb; domain=foobar.com; path=/a/b") - - c.SetExpire(CookieExpireDelete) - testCookieAppendBytes(t, c, "xxx", "yyy", "xxx=yyy; expires=Tue, 10 Nov 2009 23:00:00 GMT; domain=foobar.com; path=/a/b") -} - -func testCookieAppendBytes(t *testing.T, c *Cookie, key, value, expectedS string) { - c.SetKey(key) - c.SetValue(value) - result := string(c.AppendBytes(nil)) - if result != expectedS { - t.Fatalf("Unexpected cookie %q. Expecting %q", result, expectedS) - } -} - -func TestParseRequestCookies(t *testing.T) { - t.Parallel() - - testParseRequestCookies(t, "", "") - testParseRequestCookies(t, "=", "") - testParseRequestCookies(t, "foo", "foo") - testParseRequestCookies(t, "=foo", "foo") - testParseRequestCookies(t, "bar=", "bar=") - testParseRequestCookies(t, "xxx=aa;bb=c; =d; ;;e=g", "xxx=aa; bb=c; d; e=g") - testParseRequestCookies(t, "a;b;c; d=1;d=2", "a; b; c; d=1; d=2") - testParseRequestCookies(t, " %D0%B8%D0%B2%D0%B5%D1%82=a%20b%3Bc ;s%20s=aaa ", "%D0%B8%D0%B2%D0%B5%D1%82=a%20b%3Bc; s%20s=aaa") -} - -func testParseRequestCookies(t *testing.T, s, expectedS string) { - cookies := parseRequestCookies(nil, []byte(s)) - ss := string(appendRequestCookieBytes(nil, cookies)) - if ss != expectedS { - t.Fatalf("Unexpected cookies after parsing: %q. Expecting %q. String to parse %q", ss, expectedS, s) - } -} - -func TestAppendRequestCookieBytes(t *testing.T) { - t.Parallel() - - testAppendRequestCookieBytes(t, "=", "") - testAppendRequestCookieBytes(t, "foo=", "foo=") - testAppendRequestCookieBytes(t, "=bar", "bar") - testAppendRequestCookieBytes(t, "привет=a bc&s s=aaa", "привет=a bc; s s=aaa") -} - -func testAppendRequestCookieBytes(t *testing.T, s, expectedS string) { - kvs := strings.Split(s, "&") - cookies := make([]argsKV, 0, len(kvs)) - for _, ss := range kvs { - tmp := strings.SplitN(ss, "=", 2) - if len(tmp) != 2 { - t.Fatalf("Cannot find '=' in %q, part of %q", ss, s) - } - cookies = append(cookies, argsKV{ - key: []byte(tmp[0]), - value: []byte(tmp[1]), - }) - } - - prefix := "foobar" - result := string(appendRequestCookieBytes([]byte(prefix), cookies)) - if result[:len(prefix)] != prefix { - t.Fatalf("unexpected prefix %q. Expecting %q for cookie %q", result[:len(prefix)], prefix, s) - } - result = result[len(prefix):] - if result != expectedS { - t.Fatalf("Unexpected result %q. Expecting %q for cookie %q", result, expectedS, s) - } -} diff --git a/lib/fasthttp/cookie_timing_test.go b/lib/fasthttp/cookie_timing_test.go deleted file mode 100644 index 1af26878f..000000000 --- a/lib/fasthttp/cookie_timing_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package fasthttp - -import ( - "testing" -) - -func BenchmarkCookieParseMin(b *testing.B) { - var c Cookie - s := []byte("xxx=yyy") - for i := 0; i < b.N; i++ { - if err := c.ParseBytes(s); err != nil { - b.Fatalf("unexpected error when parsing cookies: %v", err) - } - } -} - -func BenchmarkCookieParseNoExpires(b *testing.B) { - var c Cookie - s := []byte("xxx=yyy; domain=foobar.com; path=/a/b") - for i := 0; i < b.N; i++ { - if err := c.ParseBytes(s); err != nil { - b.Fatalf("unexpected error when parsing cookies: %v", err) - } - } -} - -func BenchmarkCookieParseFull(b *testing.B) { - var c Cookie - s := []byte("xxx=yyy; expires=Tue, 10 Nov 2009 23:00:00 GMT; domain=foobar.com; path=/a/b") - for i := 0; i < b.N; i++ { - if err := c.ParseBytes(s); err != nil { - b.Fatalf("unexpected error when parsing cookies: %v", err) - } - } -} diff --git a/lib/fasthttp/expvarhandler/expvar_test.go b/lib/fasthttp/expvarhandler/expvar_test.go deleted file mode 100644 index b0fcee7ff..000000000 --- a/lib/fasthttp/expvarhandler/expvar_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package expvarhandler - -import ( - "encoding/json" - "expvar" - "strings" - "testing" - - "infini.sh/framework/lib/fasthttp" -) - -func TestExpvarHandlerBasic(t *testing.T) { - t.Parallel() - - expvar.Publish("customVar", expvar.Func(func() interface{} { - return "foobar" - })) - - var ctx fasthttp.RequestCtx - - expvarHandlerCalls.Set(0) - - ExpvarHandler(&ctx) - - body := ctx.Response.Body() - - var m map[string]interface{} - if err := json.Unmarshal(body, &m); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if _, ok := m["cmdline"]; !ok { - t.Fatalf("cannot locate cmdline expvar") - } - if _, ok := m["memstats"]; !ok { - t.Fatalf("cannot locate memstats expvar") - } - - v := m["customVar"] - sv, ok := v.(string) - if !ok { - t.Fatalf("unexpected custom var type %T. Expecting string", v) - } - if sv != "foobar" { - t.Fatalf("unexpected custom var value: %q. Expecting %q", v, "foobar") - } - - v = m["expvarHandlerCalls"] - fv, ok := v.(float64) - if !ok { - t.Fatalf("unexpected expvarHandlerCalls type %T. Expecting float64", v) - } - if int(fv) != 1 { - t.Fatalf("unexpected value for expvarHandlerCalls: %v. Expecting %v", fv, 1) - } -} - -func TestExpvarHandlerRegexp(t *testing.T) { - var ctx fasthttp.RequestCtx - ctx.QueryArgs().Set("r", "cmd") - ExpvarHandler(&ctx) - body := string(ctx.Response.Body()) - if !strings.Contains(body, `"cmdline"`) { - t.Fatalf("missing 'cmdline' expvar") - } - if strings.Contains(body, `"memstats"`) { - t.Fatalf("unexpected memstats expvar found") - } -} diff --git a/lib/fasthttp/fasthttpadaptor/adaptor_test.go b/lib/fasthttp/fasthttpadaptor/adaptor_test.go deleted file mode 100644 index c20d4e4ff..000000000 --- a/lib/fasthttp/fasthttpadaptor/adaptor_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package fasthttpadaptor - -import ( - "fmt" - "io/ioutil" - "net" - "net/http" - "net/url" - "reflect" - "testing" - - "infini.sh/framework/lib/fasthttp" -) - -func TestNewFastHTTPHandler(t *testing.T) { - t.Parallel() - - expectedMethod := fasthttp.MethodPost - expectedProto := "HTTP/1.1" - expectedProtoMajor := 1 - expectedProtoMinor := 1 - expectedRequestURI := "/foo/bar?baz=123" - expectedBody := "body 123 foo bar baz" - expectedContentLength := len(expectedBody) - expectedHost := "foobar.com" - expectedRemoteAddr := "1.2.3.4:6789" - expectedHeader := map[string]string{ - "Foo-Bar": "baz", - "Abc": "defg", - "XXX-Remote-Addr": "123.43.4543.345", - } - expectedURL, err := url.ParseRequestURI(expectedRequestURI) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - expectedContextKey := "contextKey" - expectedContextValue := "contextValue" - - callsCount := 0 - nethttpH := func(w http.ResponseWriter, r *http.Request) { - callsCount++ - if r.Method != expectedMethod { - t.Fatalf("unexpected method %q. Expecting %q", r.Method, expectedMethod) - } - if r.Proto != expectedProto { - t.Fatalf("unexpected proto %q. Expecting %q", r.Proto, expectedProto) - } - if r.ProtoMajor != expectedProtoMajor { - t.Fatalf("unexpected protoMajor %d. Expecting %d", r.ProtoMajor, expectedProtoMajor) - } - if r.ProtoMinor != expectedProtoMinor { - t.Fatalf("unexpected protoMinor %d. Expecting %d", r.ProtoMinor, expectedProtoMinor) - } - if r.RequestURI != expectedRequestURI { - t.Fatalf("unexpected requestURI %q. Expecting %q", r.RequestURI, expectedRequestURI) - } - if r.ContentLength != int64(expectedContentLength) { - t.Fatalf("unexpected contentLength %d. Expecting %d", r.ContentLength, expectedContentLength) - } - if len(r.TransferEncoding) != 0 { - t.Fatalf("unexpected transferEncoding %q. Expecting []", r.TransferEncoding) - } - if r.Host != expectedHost { - t.Fatalf("unexpected host %q. Expecting %q", r.Host, expectedHost) - } - if r.RemoteAddr != expectedRemoteAddr { - t.Fatalf("unexpected remoteAddr %q. Expecting %q", r.RemoteAddr, expectedRemoteAddr) - } - body, err := ioutil.ReadAll(r.Body) - r.Body.Close() - if err != nil { - t.Fatalf("unexpected error when reading request body: %v", err) - } - if string(body) != expectedBody { - t.Fatalf("unexpected body %q. Expecting %q", body, expectedBody) - } - if !reflect.DeepEqual(r.URL, expectedURL) { - t.Fatalf("unexpected URL: %#v. Expecting %#v", r.URL, expectedURL) - } - if r.Context().Value(expectedContextKey) != expectedContextValue { - t.Fatalf("unexpected context value for key %q. Expecting %q", expectedContextKey, expectedContextValue) - } - - for k, expectedV := range expectedHeader { - v := r.Header.Get(k) - if v != expectedV { - t.Fatalf("unexpected header value %q for key %q. Expecting %q", v, k, expectedV) - } - } - - w.Header().Set("Header1", "value1") - w.Header().Set("Header2", "value2") - w.WriteHeader(http.StatusBadRequest) - fmt.Fprintf(w, "request body is %q", body) - } - fasthttpH := NewFastHTTPHandler(http.HandlerFunc(nethttpH)) - fasthttpH = setContextValueMiddleware(fasthttpH, expectedContextKey, expectedContextValue) - - var ctx fasthttp.RequestCtx - var req fasthttp.Request - - req.Header.SetMethod(expectedMethod) - req.SetRequestURI(expectedRequestURI) - req.Header.SetHost(expectedHost) - req.BodyWriter().Write([]byte(expectedBody)) // nolint:errcheck - for k, v := range expectedHeader { - req.Header.Set(k, v) - } - - remoteAddr, err := net.ResolveTCPAddr("tcp", expectedRemoteAddr) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - ctx.Init(&req, remoteAddr, nil) - - fasthttpH(&ctx) - - if callsCount != 1 { - t.Fatalf("unexpected callsCount: %d. Expecting 1", callsCount) - } - - resp := &ctx.Response - if resp.StatusCode() != fasthttp.StatusBadRequest { - t.Fatalf("unexpected statusCode: %d. Expecting %d", resp.StatusCode(), fasthttp.StatusBadRequest) - } - if string(resp.Header.Peek("Header1")) != "value1" { - t.Fatalf("unexpected header value: %q. Expecting %q", resp.Header.Peek("Header1"), "value1") - } - if string(resp.Header.Peek("Header2")) != "value2" { - t.Fatalf("unexpected header value: %q. Expecting %q", resp.Header.Peek("Header2"), "value2") - } - expectedResponseBody := fmt.Sprintf("request body is %q", expectedBody) - if string(resp.Body()) != expectedResponseBody { - t.Fatalf("unexpected response body %q. Expecting %q", resp.Body(), expectedResponseBody) - } -} - -func setContextValueMiddleware(next fasthttp.RequestHandler, key string, value interface{}) fasthttp.RequestHandler { - return func(ctx *fasthttp.RequestCtx) { - ctx.SetUserValue(key, value) - next(ctx) - } -} - -func TestContentType(t *testing.T) { - t.Parallel() - - nethttpH := func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("")) //nolint:errcheck - } - fasthttpH := NewFastHTTPHandler(http.HandlerFunc(nethttpH)) - - var ctx fasthttp.RequestCtx - var req fasthttp.Request - - req.SetRequestURI("http://example.com") - - remoteAddr, err := net.ResolveTCPAddr("tcp", "1.2.3.4:80") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - ctx.Init(&req, remoteAddr, nil) - - fasthttpH(&ctx) - - resp := &ctx.Response - got := string(resp.Header.Peek("Content-Type")) - expected := "text/html; charset=utf-8" - if got != expected { - t.Errorf("expected %q got %q", expected, got) - } -} diff --git a/lib/fasthttp/fasthttputil/inmemory_listener_test.go b/lib/fasthttp/fasthttputil/inmemory_listener_test.go deleted file mode 100644 index 907a8d0ff..000000000 --- a/lib/fasthttp/fasthttputil/inmemory_listener_test.go +++ /dev/null @@ -1,192 +0,0 @@ -package fasthttputil - -import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "sync" - "testing" - "time" -) - -func TestInmemoryListener(t *testing.T) { - t.Parallel() - - ln := NewInmemoryListener() - - ch := make(chan struct{}) - for i := 0; i < 10; i++ { - go func(n int) { - conn, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - defer conn.Close() - req := fmt.Sprintf("request_%d", n) - nn, err := conn.Write([]byte(req)) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if nn != len(req) { - t.Errorf("unexpected number of bytes written: %d. Expecting %d", nn, len(req)) - } - buf := make([]byte, 30) - nn, err = conn.Read(buf) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - buf = buf[:nn] - resp := fmt.Sprintf("response_%d", n) - if nn != len(resp) { - t.Errorf("unexpected number of bytes read: %d. Expecting %d", nn, len(resp)) - } - if string(buf) != resp { - t.Errorf("unexpected response %q. Expecting %q", buf, resp) - } - ch <- struct{}{} - }(i) - } - - serverCh := make(chan struct{}) - go func() { - for { - conn, err := ln.Accept() - if err != nil { - close(serverCh) - return - } - defer conn.Close() - buf := make([]byte, 30) - n, err := conn.Read(buf) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - buf = buf[:n] - if !bytes.HasPrefix(buf, []byte("request_")) { - t.Errorf("unexpected request prefix %q. Expecting %q", buf, "request_") - } - resp := fmt.Sprintf("response_%s", buf[len("request_"):]) - n, err = conn.Write([]byte(resp)) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if n != len(resp) { - t.Errorf("unexpected number of bytes written: %d. Expecting %d", n, len(resp)) - } - } - }() - - for i := 0; i < 10; i++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } -} - -// echoServerHandler implements http.Handler. -type echoServerHandler struct { - t *testing.T -} - -func (s *echoServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - time.Sleep(time.Millisecond * 100) - if _, err := io.Copy(w, r.Body); err != nil { - s.t.Fatalf("unexpected error: %v", err) - } -} - -func testInmemoryListenerHTTP(t *testing.T, f func(t *testing.T, client *http.Client)) { - ln := NewInmemoryListener() - defer ln.Close() - - client := &http.Client{ - Transport: &http.Transport{ - DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { - return ln.Dial() - }, - }, - Timeout: time.Second, - } - - server := &http.Server{ - Handler: &echoServerHandler{t}, - } - - go func() { - if err := server.Serve(ln); err != nil && err != http.ErrServerClosed { - t.Errorf("unexpected error: %v", err) - } - }() - - f(t, client) - - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*100) - defer cancel() - server.Shutdown(ctx) //nolint:errcheck -} - -func testInmemoryListenerHTTPSingle(t *testing.T, client *http.Client, content string) { - res, err := client.Post("http://...", "text/plain", bytes.NewBufferString(content)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - b, err := ioutil.ReadAll(res.Body) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - s := string(b) - if string(b) != content { - t.Fatalf("unexpected response %q, expecting %q", s, content) - } -} - -func TestInmemoryListenerHTTPSingle(t *testing.T) { - t.Parallel() - - testInmemoryListenerHTTP(t, func(t *testing.T, client *http.Client) { - testInmemoryListenerHTTPSingle(t, client, "request") - }) -} - -func TestInmemoryListenerHTTPSerial(t *testing.T) { - t.Parallel() - - testInmemoryListenerHTTP(t, func(t *testing.T, client *http.Client) { - for i := 0; i < 10; i++ { - testInmemoryListenerHTTPSingle(t, client, fmt.Sprintf("request_%d", i)) - } - }) -} - -func TestInmemoryListenerHTTPConcurrent(t *testing.T) { - t.Parallel() - - testInmemoryListenerHTTP(t, func(t *testing.T, client *http.Client) { - var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - testInmemoryListenerHTTPSingle(t, client, fmt.Sprintf("request_%d", i)) - }(i) - } - wg.Wait() - }) -} diff --git a/lib/fasthttp/fasthttputil/inmemory_listener_timing_test.go b/lib/fasthttp/fasthttputil/inmemory_listener_timing_test.go deleted file mode 100644 index 277b46d56..000000000 --- a/lib/fasthttp/fasthttputil/inmemory_listener_timing_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package fasthttputil_test - -import ( - "crypto/tls" - "net" - "testing" - - "infini.sh/framework/lib/fasthttp" - "infini.sh/framework/lib/fasthttp/fasthttputil" -) - -// BenchmarkPlainStreaming measures end-to-end plaintext streaming performance -// for fasthttp client and server. -// -// It issues http requests over a small number of keep-alive connections. -func BenchmarkPlainStreaming(b *testing.B) { - benchmark(b, streamingHandler, false) -} - -// BenchmarkPlainHandshake measures end-to-end plaintext handshake performance -// for fasthttp client and server. -// -// It re-establishes new connection per each http request. -func BenchmarkPlainHandshake(b *testing.B) { - benchmark(b, handshakeHandler, false) -} - -// BenchmarkTLSStreaming measures end-to-end TLS streaming performance -// for fasthttp client and server. -// -// It issues http requests over a small number of TLS keep-alive connections. -func BenchmarkTLSStreaming(b *testing.B) { - benchmark(b, streamingHandler, true) -} - -func benchmark(b *testing.B, h fasthttp.RequestHandler, isTLS bool) { - var serverTLSConfig, clientTLSConfig *tls.Config - if isTLS { - certFile := "rsa.pem" - keyFile := "rsa.key" - cert, err := tls.LoadX509KeyPair(certFile, keyFile) - if err != nil { - b.Fatalf("cannot load TLS certificate from certFile=%q, keyFile=%q: %v", certFile, keyFile, err) - } - serverTLSConfig = &tls.Config{ - Certificates: []tls.Certificate{cert}, - PreferServerCipherSuites: true, - } - serverTLSConfig.CurvePreferences = []tls.CurveID{} - clientTLSConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } - ln := fasthttputil.NewInmemoryListener() - serverStopCh := make(chan struct{}) - go func() { - serverLn := net.Listener(ln) - if serverTLSConfig != nil { - serverLn = tls.NewListener(serverLn, serverTLSConfig) - } - if err := fasthttp.Serve(serverLn, h); err != nil { - b.Errorf("unexpected error in server: %v", err) - } - close(serverStopCh) - }() - c := &fasthttp.HostClient{ - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - IsTLS: isTLS, - TLSConfig: clientTLSConfig, - } - - b.RunParallel(func(pb *testing.PB) { - runRequests(b, pb, c, isTLS) - }) - ln.Close() - <-serverStopCh -} - -func streamingHandler(ctx *fasthttp.RequestCtx) { - ctx.WriteString("foobar") //nolint:errcheck -} - -func handshakeHandler(ctx *fasthttp.RequestCtx) { - streamingHandler(ctx) - - // Explicitly close connection after each response. - ctx.SetConnectionClose() -} - -func runRequests(b *testing.B, pb *testing.PB, c *fasthttp.HostClient, isTLS bool) { - var req fasthttp.Request - if isTLS { - req.SetRequestURI("https://foo.bar/baz") - } else { - req.SetRequestURI("http://foo.bar/baz") - } - var resp fasthttp.Response - for pb.Next() { - if err := c.Do(&req, &resp); err != nil { - b.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != fasthttp.StatusOK { - b.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), fasthttp.StatusOK) - } - } -} diff --git a/lib/fasthttp/fasthttputil/pipeconns_test.go b/lib/fasthttp/fasthttputil/pipeconns_test.go deleted file mode 100644 index e81b0ad43..000000000 --- a/lib/fasthttp/fasthttputil/pipeconns_test.go +++ /dev/null @@ -1,360 +0,0 @@ -package fasthttputil - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "net" - "testing" - "time" -) - -func TestPipeConnsWriteTimeout(t *testing.T) { - t.Parallel() - - pc := NewPipeConns() - c1 := pc.Conn1() - - deadline := time.Now().Add(time.Millisecond) - if err := c1.SetWriteDeadline(deadline); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - data := []byte("foobar") - for { - _, err := c1.Write(data) - if err != nil { - if err == ErrTimeout { - break - } - t.Fatalf("unexpected error: %v", err) - } - } - - for i := 0; i < 10; i++ { - _, err := c1.Write(data) - if err == nil { - t.Fatalf("expecting error") - } - if err != ErrTimeout { - t.Fatalf("unexpected error: %v. Expecting %v", err, ErrTimeout) - } - } - - // read the written data - c2 := pc.Conn2() - if err := c2.SetReadDeadline(time.Now().Add(10 * time.Millisecond)); err != nil { - t.Fatalf("unexpected error: %v", err) - } - for { - _, err := c2.Read(data) - if err != nil { - if err == ErrTimeout { - break - } - t.Fatalf("unexpected error: %v", err) - } - } - - for i := 0; i < 10; i++ { - _, err := c2.Read(data) - if err == nil { - t.Fatalf("expecting error") - } - if err != ErrTimeout { - t.Fatalf("unexpected error: %v. Expecting %v", err, ErrTimeout) - } - } -} - -func TestPipeConnsPositiveReadTimeout(t *testing.T) { - t.Parallel() - - testPipeConnsReadTimeout(t, time.Millisecond) -} - -func TestPipeConnsNegativeReadTimeout(t *testing.T) { - t.Parallel() - - testPipeConnsReadTimeout(t, -time.Second) -} - -var zeroTime time.Time - -func testPipeConnsReadTimeout(t *testing.T, timeout time.Duration) { - pc := NewPipeConns() - c1 := pc.Conn1() - - deadline := time.Now().Add(timeout) - if err := c1.SetReadDeadline(deadline); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var buf [1]byte - for i := 0; i < 10; i++ { - _, err := c1.Read(buf[:]) - if err == nil { - t.Fatalf("expecting error on iteration %d", i) - } - if err != ErrTimeout { - t.Fatalf("unexpected error on iteration %d: %v. Expecting %v", i, err, ErrTimeout) - } - } - - // disable deadline and send data from c2 to c1 - if err := c1.SetReadDeadline(zeroTime); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - data := []byte("foobar") - c2 := pc.Conn2() - if _, err := c2.Write(data); err != nil { - t.Fatalf("unexpected error: %v", err) - } - dataBuf := make([]byte, len(data)) - if _, err := io.ReadFull(c1, dataBuf); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !bytes.Equal(data, dataBuf) { - t.Fatalf("unexpected data received: %q. Expecting %q", dataBuf, data) - } -} - -func TestPipeConnsCloseWhileReadWriteConcurrent(t *testing.T) { - t.Parallel() - - concurrency := 4 - ch := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - testPipeConnsCloseWhileReadWriteSerial(t) - ch <- struct{}{} - }() - } - - for i := 0; i < concurrency; i++ { - select { - case <-ch: - case <-time.After(5 * time.Second): - t.Fatalf("timeout") - } - } -} - -func TestPipeConnsCloseWhileReadWriteSerial(t *testing.T) { - t.Parallel() - - testPipeConnsCloseWhileReadWriteSerial(t) -} - -func testPipeConnsCloseWhileReadWriteSerial(t *testing.T) { - for i := 0; i < 10; i++ { - testPipeConnsCloseWhileReadWrite(t) - } -} - -func testPipeConnsCloseWhileReadWrite(t *testing.T) { - pc := NewPipeConns() - c1 := pc.Conn1() - c2 := pc.Conn2() - - readCh := make(chan error) - go func() { - var err error - if _, err = io.Copy(ioutil.Discard, c1); err != nil { - if err != errConnectionClosed { - err = fmt.Errorf("unexpected error: %w", err) - } else { - err = nil - } - } - readCh <- err - }() - - writeCh := make(chan error) - go func() { - var err error - for { - if _, err = c2.Write([]byte("foobar")); err != nil { - if err != errConnectionClosed { - err = fmt.Errorf("unexpected error: %w", err) - } else { - err = nil - } - break - } - } - writeCh <- err - }() - - time.Sleep(10 * time.Millisecond) - if err := c1.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := c2.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case err := <-readCh: - if err != nil { - t.Fatalf("unexpected error in reader: %v", err) - } - case <-time.After(time.Second): - t.Fatalf("timeout") - } - select { - case err := <-writeCh: - if err != nil { - t.Fatalf("unexpected error in writer: %v", err) - } - case <-time.After(time.Second): - t.Fatalf("timeout") - } -} - -func TestPipeConnsReadWriteSerial(t *testing.T) { - t.Parallel() - - testPipeConnsReadWriteSerial(t) -} - -func TestPipeConnsReadWriteConcurrent(t *testing.T) { - t.Parallel() - - testConcurrency(t, 10, testPipeConnsReadWriteSerial) -} - -func testPipeConnsReadWriteSerial(t *testing.T) { - pc := NewPipeConns() - testPipeConnsReadWrite(t, pc.Conn1(), pc.Conn2()) - - pc = NewPipeConns() - testPipeConnsReadWrite(t, pc.Conn2(), pc.Conn1()) -} - -func testPipeConnsReadWrite(t *testing.T, c1, c2 net.Conn) { - defer c1.Close() - defer c2.Close() - - var buf [32]byte - for i := 0; i < 10; i++ { - // The first write - s1 := fmt.Sprintf("foo_%d", i) - n, err := c1.Write([]byte(s1)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if n != len(s1) { - t.Fatalf("unexpected number of bytes written: %d. Expecting %d", n, len(s1)) - } - - // The second write - s2 := fmt.Sprintf("bar_%d", i) - n, err = c1.Write([]byte(s2)) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if n != len(s2) { - t.Fatalf("unexpected number of bytes written: %d. Expecting %d", n, len(s2)) - } - - // Read data written above in two writes - s := s1 + s2 - n, err = c2.Read(buf[:]) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if n != len(s) { - t.Fatalf("unexpected number of bytes read: %d. Expecting %d", n, len(s)) - } - if string(buf[:n]) != s { - t.Fatalf("unexpected string read: %q. Expecting %q", buf[:n], s) - } - } -} - -func TestPipeConnsCloseSerial(t *testing.T) { - t.Parallel() - - testPipeConnsCloseSerial(t) -} - -func TestPipeConnsCloseConcurrent(t *testing.T) { - t.Parallel() - - testConcurrency(t, 10, testPipeConnsCloseSerial) -} - -func testPipeConnsCloseSerial(t *testing.T) { - pc := NewPipeConns() - testPipeConnsClose(t, pc.Conn1(), pc.Conn2()) - - pc = NewPipeConns() - testPipeConnsClose(t, pc.Conn2(), pc.Conn1()) -} - -func testPipeConnsClose(t *testing.T, c1, c2 net.Conn) { - if err := c1.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - var buf [10]byte - - // attempt writing to closed conn - for i := 0; i < 10; i++ { - n, err := c1.Write(buf[:]) - if err == nil { - t.Fatalf("expecting error") - } - if n != 0 { - t.Fatalf("unexpected number of bytes written: %d. Expecting 0", n) - } - } - - // attempt reading from closed conn - for i := 0; i < 10; i++ { - n, err := c2.Read(buf[:]) - if err == nil { - t.Fatalf("expecting error") - } - if err != io.EOF { - t.Fatalf("unexpected error: %v. Expecting %v", err, io.EOF) - } - if n != 0 { - t.Fatalf("unexpected number of bytes read: %d. Expecting 0", n) - } - } - - if err := c2.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // attempt closing already closed conns - for i := 0; i < 10; i++ { - if err := c1.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := c2.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } -} - -func testConcurrency(t *testing.T, concurrency int, f func(*testing.T)) { - ch := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - f(t) - ch <- struct{}{} - }() - } - - for i := 0; i < concurrency; i++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } -} diff --git a/lib/fasthttp/fs_example_test.go b/lib/fasthttp/fs_example_test.go deleted file mode 100644 index 9027e76bb..000000000 --- a/lib/fasthttp/fs_example_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package fasthttp_test - -import ( - "log" - - "infini.sh/framework/lib/fasthttp" -) - -func ExampleFS() { - fs := &fasthttp.FS{ - // Path to directory to serve. - Root: "/var/www/static-site", - - // Generate index pages if client requests directory contents. - GenerateIndexPages: true, - - // Enable transparent compression to save network traffic. - Compress: true, - } - - // Create request handler for serving static files. - h := fs.NewRequestHandler() - - // Start the server. - if err := fasthttp.ListenAndServe(":8080", h); err != nil { - log.Fatalf("error in ListenAndServe: %v", err) - } -} diff --git a/lib/fasthttp/fs_handler_example_test.go b/lib/fasthttp/fs_handler_example_test.go deleted file mode 100644 index 81eeaae2c..000000000 --- a/lib/fasthttp/fs_handler_example_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package fasthttp_test - -import ( - "bytes" - "log" - - "infini.sh/framework/lib/fasthttp" -) - -// Setup file handlers (aka 'file server config') -var ( - // Handler for serving images from /img/ path, - // i.e. /img/foo/bar.jpg will be served from - // /var/www/images/foo/bar.jpb . - imgPrefix = []byte("/img/") - imgHandler = fasthttp.FSHandler("/var/www/images", 1) - - // Handler for serving css from /static/css/ path, - // i.e. /static/css/foo/bar.css will be served from - // /home/dev/css/foo/bar.css . - cssPrefix = []byte("/static/css/") - cssHandler = fasthttp.FSHandler("/home/dev/css", 2) - - // Handler for serving the rest of requests, - // i.e. /foo/bar/baz.html will be served from - // /var/www/files/foo/bar/baz.html . - filesHandler = fasthttp.FSHandler("/var/www/files", 0) -) - -// Main request handler -func requestHandler(ctx *fasthttp.RequestCtx) { - path := ctx.Path() - switch { - case bytes.HasPrefix(path, imgPrefix): - imgHandler(ctx) - case bytes.HasPrefix(path, cssPrefix): - cssHandler(ctx) - default: - filesHandler(ctx) - } -} - -func ExampleFSHandler() { - if err := fasthttp.ListenAndServe(":80", requestHandler); err != nil { - log.Fatalf("Error in server: %v", err) - } -} diff --git a/lib/fasthttp/fs_test.go b/lib/fasthttp/fs_test.go deleted file mode 100644 index 75178570f..000000000 --- a/lib/fasthttp/fs_test.go +++ /dev/null @@ -1,853 +0,0 @@ -// go:build !windows -// Don't run FS tests on windows as it isn't compatible for now. - -package fasthttp - -import ( - "bufio" - "bytes" - "fmt" - "io" - "io/ioutil" - "math/rand" - "os" - "path" - "runtime" - "sort" - "testing" - "time" -) - -type TestLogger struct { - t *testing.T -} - -func (t TestLogger) Printf(format string, args ...interface{}) { - t.t.Logf(format, args...) -} - -func TestNewVHostPathRewriter(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - var req Request - req.Header.SetHost("foobar.com") - req.SetRequestURI("/foo/bar/baz") - ctx.Init(&req, nil, nil) - - f := NewVHostPathRewriter(0) - path := f(&ctx) - expectedPath := "/foobar.com/foo/bar/baz" - if string(path) != expectedPath { - t.Fatalf("unexpected path %q. Expecting %q", path, expectedPath) - } - - ctx.Request.Reset() - ctx.Request.SetRequestURI("https://aaa.bbb.cc/one/two/three/four?asdf=dsf") - f = NewVHostPathRewriter(2) - path = f(&ctx) - expectedPath = "/aaa.bbb.cc/three/four" - if string(path) != expectedPath { - t.Fatalf("unexpected path %q. Expecting %q", path, expectedPath) - } -} - -func TestNewVHostPathRewriterMaliciousHost(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - var req Request - req.Header.SetHost("/../../../etc/passwd") - req.SetRequestURI("/foo/bar/baz") - ctx.Init(&req, nil, nil) - - f := NewVHostPathRewriter(0) - path := f(&ctx) - expectedPath := "/invalid-host/" - if string(path) != expectedPath { - t.Fatalf("unexpected path %q. Expecting %q", path, expectedPath) - } -} - -func testPathNotFound(t *testing.T, pathNotFoundFunc RequestHandler) { - var ctx RequestCtx - var req Request - req.SetRequestURI("http//some.url/file") - ctx.Init(&req, nil, TestLogger{t}) - - stop := make(chan struct{}) - defer close(stop) - - fs := &FS{ - Root: "./", - PathNotFound: pathNotFoundFunc, - CleanStop: stop, - } - fs.NewRequestHandler()(&ctx) - - if pathNotFoundFunc == nil { - // different to ... - if !bytes.Equal(ctx.Response.Body(), - []byte("Cannot open requested path")) { - t.Fatalf("response defers. Response: %q", ctx.Response.Body()) - } - } else { - // Equals to ... - if bytes.Equal(ctx.Response.Body(), - []byte("Cannot open requested path")) { - t.Fatalf("response defers. Response: %q", ctx.Response.Body()) - } - } -} - -func TestPathNotFound(t *testing.T) { - t.Parallel() - - testPathNotFound(t, nil) -} - -func TestPathNotFoundFunc(t *testing.T) { - t.Parallel() - - testPathNotFound(t, func(ctx *RequestCtx) { - ctx.WriteString("Not found hehe") //nolint:errcheck - }) -} - -func TestServeFileHead(t *testing.T) { - // This test can't run parallel as files in / might by changed by other tests. - - var ctx RequestCtx - var req Request - req.Header.SetMethod(MethodHead) - req.SetRequestURI("http://foobar.com/baz") - ctx.Init(&req, nil, nil) - - ServeFile(&ctx, "fs.go") - - var resp Response - resp.SkipBody = true - s := ctx.Response.String() - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - ce := resp.Header.ContentEncoding() - if len(ce) > 0 { - t.Fatalf("Unexpected 'Content-Encoding' %q", ce) - } - - body := resp.Body() - if len(body) > 0 { - t.Fatalf("unexpected response body %q. Expecting empty body", body) - } - - expectedBody, err := getFileContents("/fs.go") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - contentLength := resp.Header.ContentLength() - if contentLength != len(expectedBody) { - t.Fatalf("unexpected Content-Length: %d. expecting %d", contentLength, len(expectedBody)) - } -} - -func TestServeFileSmallNoReadFrom(t *testing.T) { - t.Parallel() - - teststr := "hello, world!" - - tempdir, err := ioutil.TempDir("", "httpexpect") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tempdir) - - if err := ioutil.WriteFile( - path.Join(tempdir, "hello"), []byte(teststr), 0666); err != nil { - t.Fatal(err) - } - - var ctx RequestCtx - var req Request - req.SetRequestURI("http://foobar.com/baz") - ctx.Init(&req, nil, nil) - - ServeFile(&ctx, path.Join(tempdir, "hello")) - - reader, ok := ctx.Response.bodyStream.(*fsSmallFileReader) - if !ok { - t.Fatal("expected fsSmallFileReader") - } - - buf := bytes.NewBuffer(nil) - - n, err := reader.WriteTo(pureWriter{buf}) - if err != nil { - t.Fatal(err) - } - - if n != int64(len(teststr)) { - t.Fatalf("expected %d bytes, got %d bytes", len(teststr), n) - } - - body := buf.String() - if body != teststr { - t.Fatalf("expected '%q'", teststr) - } -} - -type pureWriter struct { - w io.Writer -} - -func (pw pureWriter) Write(p []byte) (nn int, err error) { - return pw.w.Write(p) -} - -func TestServeFileCompressed(t *testing.T) { - // This test can't run parallel as files in / might by changed by other tests. - - var ctx RequestCtx - ctx.Init(&Request{}, nil, nil) - - var resp Response - - // request compressed gzip file - ctx.Request.SetRequestURI("http://foobar.com/baz") - ctx.Request.Header.Set(HeaderAcceptEncoding, "gzip") - ServeFile(&ctx, "fs.go") - - s := ctx.Response.String() - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - ce := resp.Header.ContentEncoding() - if string(ce) != "gzip" { - t.Fatalf("Unexpected 'Content-Encoding' %q. Expecting %q", ce, "gzip") - } - - body, err := resp.BodyGunzip() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - expectedBody, err := getFileContents("/fs.go") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !bytes.Equal(body, expectedBody) { - t.Fatalf("unexpected body %q. expecting %q", body, expectedBody) - } - - // request compressed brotli file - ctx.Request.Reset() - ctx.Request.SetRequestURI("http://foobar.com/baz") - ctx.Request.Header.Set(HeaderAcceptEncoding, "br") - ServeFile(&ctx, "fs.go") - - s = ctx.Response.String() - br = bufio.NewReader(bytes.NewBufferString(s)) - if err = resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - ce = resp.Header.ContentEncoding() - if string(ce) != "br" { - t.Fatalf("Unexpected 'Content-Encoding' %q. Expecting %q", ce, "br") - } - - body, err = resp.BodyUnbrotli() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - expectedBody, err = getFileContents("/fs.go") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !bytes.Equal(body, expectedBody) { - t.Fatalf("unexpected body %q. expecting %q", body, expectedBody) - } -} - -func TestServeFileUncompressed(t *testing.T) { - // This test can't run parallel as files in / might by changed by other tests. - - var ctx RequestCtx - var req Request - req.SetRequestURI("http://foobar.com/baz") - req.Header.Set(HeaderAcceptEncoding, "gzip") - ctx.Init(&req, nil, nil) - - ServeFileUncompressed(&ctx, "fs.go") - - var resp Response - s := ctx.Response.String() - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - ce := resp.Header.ContentEncoding() - if len(ce) > 0 { - t.Fatalf("Unexpected 'Content-Encoding' %q", ce) - } - - body := resp.Body() - expectedBody, err := getFileContents("/fs.go") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !bytes.Equal(body, expectedBody) { - t.Fatalf("unexpected body %q. expecting %q", body, expectedBody) - } -} - -func TestFSByteRangeConcurrent(t *testing.T) { - // This test can't run parallel as files in / might by changed by other tests. - - stop := make(chan struct{}) - defer close(stop) - - fs := &FS{ - Root: ".", - AcceptByteRange: true, - CleanStop: stop, - } - h := fs.NewRequestHandler() - - concurrency := 10 - ch := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - for j := 0; j < 5; j++ { - testFSByteRange(t, h, "/fs.go") - testFSByteRange(t, h, "/README.md") - } - ch <- struct{}{} - }() - } - - for i := 0; i < concurrency; i++ { - select { - case <-time.After(time.Second): - t.Fatalf("timeout") - case <-ch: - } - } -} - -func TestFSByteRangeSingleThread(t *testing.T) { - // This test can't run parallel as files in / might by changed by other tests. - - stop := make(chan struct{}) - defer close(stop) - - fs := &FS{ - Root: ".", - AcceptByteRange: true, - CleanStop: stop, - } - h := fs.NewRequestHandler() - - testFSByteRange(t, h, "/fs.go") - testFSByteRange(t, h, "/README.md") -} - -func testFSByteRange(t *testing.T, h RequestHandler, filePath string) { - var ctx RequestCtx - ctx.Init(&Request{}, nil, nil) - - expectedBody, err := getFileContents(filePath) - if err != nil { - t.Fatalf("cannot read file %q: %v", filePath, err) - } - - fileSize := len(expectedBody) - startPos := rand.Intn(fileSize) - endPos := rand.Intn(fileSize) - if endPos < startPos { - startPos, endPos = endPos, startPos - } - - ctx.Request.SetRequestURI(filePath) - ctx.Request.Header.SetByteRange(startPos, endPos) - h(&ctx) - - var resp Response - s := ctx.Response.String() - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v. filePath=%q", err, filePath) - } - if resp.StatusCode() != StatusPartialContent { - t.Fatalf("unexpected status code: %d. Expecting %d. filePath=%q", resp.StatusCode(), StatusPartialContent, filePath) - } - cr := resp.Header.Peek(HeaderContentRange) - - expectedCR := fmt.Sprintf("bytes %d-%d/%d", startPos, endPos, fileSize) - if string(cr) != expectedCR { - t.Fatalf("unexpected content-range %q. Expecting %q. filePath=%q", cr, expectedCR, filePath) - } - body := resp.Body() - bodySize := endPos - startPos + 1 - if len(body) != bodySize { - t.Fatalf("unexpected body size %d. Expecting %d. filePath=%q, startPos=%d, endPos=%d", - len(body), bodySize, filePath, startPos, endPos) - } - - expectedBody = expectedBody[startPos : endPos+1] - if !bytes.Equal(body, expectedBody) { - t.Fatalf("unexpected body %q. Expecting %q. filePath=%q, startPos=%d, endPos=%d", - body, expectedBody, filePath, startPos, endPos) - } -} - -func getFileContents(path string) ([]byte, error) { - path = "." + path - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - return ioutil.ReadAll(f) -} - -func TestParseByteRangeSuccess(t *testing.T) { - t.Parallel() - - testParseByteRangeSuccess(t, "bytes=0-0", 1, 0, 0) - testParseByteRangeSuccess(t, "bytes=1234-6789", 6790, 1234, 6789) - - testParseByteRangeSuccess(t, "bytes=123-", 456, 123, 455) - testParseByteRangeSuccess(t, "bytes=-1", 1, 0, 0) - testParseByteRangeSuccess(t, "bytes=-123", 456, 333, 455) - - // End position exceeding content-length. It should be updated to content-length-1. - // See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 - testParseByteRangeSuccess(t, "bytes=1-2345", 234, 1, 233) - testParseByteRangeSuccess(t, "bytes=0-2345", 2345, 0, 2344) - - // Start position overflow. Whole range must be returned. - // See https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35 - testParseByteRangeSuccess(t, "bytes=-567", 56, 0, 55) -} - -func testParseByteRangeSuccess(t *testing.T, v string, contentLength, startPos, endPos int) { - startPos1, endPos1, err := ParseByteRange([]byte(v), contentLength) - if err != nil { - t.Fatalf("unexpected error: %v. v=%q, contentLength=%d", err, v, contentLength) - } - if startPos1 != startPos { - t.Fatalf("unexpected startPos=%d. Expecting %d. v=%q, contentLength=%d", startPos1, startPos, v, contentLength) - } - if endPos1 != endPos { - t.Fatalf("unexpected endPos=%d. Expectind %d. v=%q, contentLenght=%d", endPos1, endPos, v, contentLength) - } -} - -func TestParseByteRangeError(t *testing.T) { - t.Parallel() - - // invalid value - testParseByteRangeError(t, "asdfasdfas", 1234) - - // invalid units - testParseByteRangeError(t, "foobar=1-34", 600) - - // missing '-' - testParseByteRangeError(t, "bytes=1234", 1235) - - // non-numeric range - testParseByteRangeError(t, "bytes=foobar", 123) - testParseByteRangeError(t, "bytes=1-foobar", 123) - testParseByteRangeError(t, "bytes=df-344", 545) - - // multiple byte ranges - testParseByteRangeError(t, "bytes=1-2,4-6", 123) - - // byte range exceeding contentLength - testParseByteRangeError(t, "bytes=123-", 12) - - // startPos exceeding endPos - testParseByteRangeError(t, "bytes=123-34", 1234) -} - -func testParseByteRangeError(t *testing.T, v string, contentLength int) { - _, _, err := ParseByteRange([]byte(v), contentLength) - if err == nil { - t.Fatalf("expecting error when parsing byte range %q", v) - } -} - -func TestFSCompressConcurrent(t *testing.T) { - // Don't run this test on Windows, the Windows Github actions are to slow and timeout too often. - if runtime.GOOS == "windows" { - t.SkipNow() - } - - // This test can't run parallel as files in / might be changed by other tests. - - stop := make(chan struct{}) - defer close(stop) - - fs := &FS{ - Root: ".", - GenerateIndexPages: true, - Compress: true, - CompressBrotli: true, - CleanStop: stop, - } - h := fs.NewRequestHandler() - - concurrency := 4 - ch := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - for j := 0; j < 5; j++ { - testFSCompress(t, h, "/fs.go") - testFSCompress(t, h, "/") - testFSCompress(t, h, "/README.md") - } - ch <- struct{}{} - }() - } - - for i := 0; i < concurrency; i++ { - select { - case <-ch: - case <-time.After(time.Second * 2): - t.Fatalf("timeout") - } - } -} - -func TestFSCompressSingleThread(t *testing.T) { - // This test can't run parallel as files in / might by changed by other tests. - - stop := make(chan struct{}) - defer close(stop) - - fs := &FS{ - Root: ".", - GenerateIndexPages: true, - Compress: true, - CompressBrotli: true, - CleanStop: stop, - } - h := fs.NewRequestHandler() - - testFSCompress(t, h, "/fs.go") - testFSCompress(t, h, "/") - testFSCompress(t, h, "/README.md") -} - -func testFSCompress(t *testing.T, h RequestHandler, filePath string) { - // File locking is flaky on Windows. - if runtime.GOOS == "windows" { - t.SkipNow() - } - - var ctx RequestCtx - ctx.Init(&Request{}, nil, nil) - - var resp Response - - // request uncompressed file - ctx.Request.Reset() - ctx.Request.SetRequestURI(filePath) - h(&ctx) - s := ctx.Response.String() - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Errorf("unexpected error: %v. filePath=%q", err, filePath) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d. filePath=%q", resp.StatusCode(), StatusOK, filePath) - } - ce := resp.Header.ContentEncoding() - if string(ce) != "" { - t.Errorf("unexpected content-encoding %q. Expecting empty string. filePath=%q", ce, filePath) - } - body := string(resp.Body()) - - // request compressed gzip file - ctx.Request.Reset() - ctx.Request.SetRequestURI(filePath) - ctx.Request.Header.Set(HeaderAcceptEncoding, "gzip") - h(&ctx) - s = ctx.Response.String() - br = bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Errorf("unexpected error: %v. filePath=%q", err, filePath) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d. filePath=%q", resp.StatusCode(), StatusOK, filePath) - } - ce = resp.Header.ContentEncoding() - if string(ce) != "gzip" { - t.Errorf("unexpected content-encoding %q. Expecting %q. filePath=%q", ce, "gzip", filePath) - } - zbody, err := resp.BodyGunzip() - if err != nil { - t.Errorf("unexpected error when gunzipping response body: %v. filePath=%q", err, filePath) - } - if string(zbody) != body { - t.Errorf("unexpected body len=%d. Expected len=%d. FilePath=%q", len(zbody), len(body), filePath) - } - - // request compressed brotli file - ctx.Request.Reset() - ctx.Request.SetRequestURI(filePath) - ctx.Request.Header.Set(HeaderAcceptEncoding, "br") - h(&ctx) - s = ctx.Response.String() - br = bufio.NewReader(bytes.NewBufferString(s)) - if err = resp.Read(br); err != nil { - t.Errorf("unexpected error: %v. filePath=%q", err, filePath) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d. filePath=%q", resp.StatusCode(), StatusOK, filePath) - } - ce = resp.Header.ContentEncoding() - if string(ce) != "br" { - t.Errorf("unexpected content-encoding %q. Expecting %q. filePath=%q", ce, "br", filePath) - } - zbody, err = resp.BodyUnbrotli() - if err != nil { - t.Errorf("unexpected error when unbrotling response body: %v. filePath=%q", err, filePath) - } - if string(zbody) != body { - t.Errorf("unexpected body len=%d. Expected len=%d. FilePath=%q", len(zbody), len(body), filePath) - } -} - -func TestFSHandlerSingleThread(t *testing.T) { - // This test can't run parallel as files in / might by changed by other tests. - - requestHandler := FSHandler(".", 0) - - f, err := os.Open(".") - if err != nil { - t.Fatalf("cannot open cwd: %v", err) - } - - filenames, err := f.Readdirnames(0) - f.Close() - if err != nil { - t.Fatalf("cannot read dirnames in cwd: %v", err) - } - sort.Strings(filenames) - - for i := 0; i < 3; i++ { - fsHandlerTest(t, requestHandler, filenames) - } -} - -func TestFSHandlerConcurrent(t *testing.T) { - // This test can't run parallel as files in / might by changed by other tests. - - requestHandler := FSHandler(".", 0) - - f, err := os.Open(".") - if err != nil { - t.Fatalf("cannot open cwd: %v", err) - } - - filenames, err := f.Readdirnames(0) - f.Close() - if err != nil { - t.Fatalf("cannot read dirnames in cwd: %v", err) - } - sort.Strings(filenames) - - concurrency := 10 - ch := make(chan struct{}, concurrency) - for j := 0; j < concurrency; j++ { - go func() { - for i := 0; i < 3; i++ { - fsHandlerTest(t, requestHandler, filenames) - } - ch <- struct{}{} - }() - } - - for j := 0; j < concurrency; j++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } -} - -func fsHandlerTest(t *testing.T, requestHandler RequestHandler, filenames []string) { - var ctx RequestCtx - var req Request - ctx.Init(&req, nil, nil) - ctx.Request.Header.SetHost("foobar.com") - - filesTested := 0 - for _, name := range filenames { - f, err := os.Open(name) - if err != nil { - t.Fatalf("cannot open file %q: %v", name, err) - } - stat, err := f.Stat() - if err != nil { - t.Fatalf("cannot get file stat %q: %v", name, err) - } - if stat.IsDir() { - f.Close() - continue - } - data, err := ioutil.ReadAll(f) - f.Close() - if err != nil { - t.Fatalf("cannot read file contents %q: %v", name, err) - } - - ctx.PhantomURI().Update(name) - requestHandler(&ctx) - if ctx.Response.bodyStream == nil { - t.Fatalf("response body stream must be non-empty") - } - body, err := ioutil.ReadAll(ctx.Response.bodyStream) - if err != nil { - t.Fatalf("error when reading response body stream: %v", err) - } - if !bytes.Equal(body, data) { - t.Fatalf("unexpected body returned: %q. Expecting %q", body, data) - } - filesTested++ - if filesTested >= 10 { - break - } - } - - // verify index page generation - ctx.PhantomURI().Update("/") - requestHandler(&ctx) - if ctx.Response.bodyStream == nil { - t.Fatalf("response body stream must be non-empty") - } - body, err := ioutil.ReadAll(ctx.Response.bodyStream) - if err != nil { - t.Fatalf("error when reading response body stream: %v", err) - } - if len(body) == 0 { - t.Fatalf("index page must be non-empty") - } -} - -func TestStripPathSlashes(t *testing.T) { - t.Parallel() - - testStripPathSlashes(t, "", 0, "") - testStripPathSlashes(t, "", 10, "") - testStripPathSlashes(t, "/", 0, "") - testStripPathSlashes(t, "/", 1, "") - testStripPathSlashes(t, "/", 10, "") - testStripPathSlashes(t, "/foo/bar/baz", 0, "/foo/bar/baz") - testStripPathSlashes(t, "/foo/bar/baz", 1, "/bar/baz") - testStripPathSlashes(t, "/foo/bar/baz", 2, "/baz") - testStripPathSlashes(t, "/foo/bar/baz", 3, "") - testStripPathSlashes(t, "/foo/bar/baz", 10, "") - - // trailing slash - testStripPathSlashes(t, "/foo/bar/", 0, "/foo/bar") - testStripPathSlashes(t, "/foo/bar/", 1, "/bar") - testStripPathSlashes(t, "/foo/bar/", 2, "") - testStripPathSlashes(t, "/foo/bar/", 3, "") -} - -func testStripPathSlashes(t *testing.T, path string, stripSlashes int, expectedPath string) { - s := stripLeadingSlashes([]byte(path), stripSlashes) - s = stripTrailingSlashes(s) - if string(s) != expectedPath { - t.Fatalf("unexpected path after stripping %q with stripSlashes=%d: %q. Expecting %q", path, stripSlashes, s, expectedPath) - } -} - -func TestFileExtension(t *testing.T) { - t.Parallel() - - testFileExtension(t, "foo.bar", false, "zzz", ".bar") - testFileExtension(t, "foobar", false, "zzz", "") - testFileExtension(t, "foo.bar.baz", false, "zzz", ".baz") - testFileExtension(t, "", false, "zzz", "") - testFileExtension(t, "/a/b/c.d/efg.jpg", false, ".zzz", ".jpg") - - testFileExtension(t, "foo.bar", true, ".zzz", ".bar") - testFileExtension(t, "foobar.zzz", true, ".zzz", "") - testFileExtension(t, "foo.bar.baz.fasthttp.gz", true, ".fasthttp.gz", ".baz") - testFileExtension(t, "", true, ".zzz", "") - testFileExtension(t, "/a/b/c.d/efg.jpg.xxx", true, ".xxx", ".jpg") -} - -func testFileExtension(t *testing.T, path string, compressed bool, compressedFileSuffix, expectedExt string) { - ext := fileExtension(path, compressed, compressedFileSuffix) - if ext != expectedExt { - t.Fatalf("unexpected file extension for file %q: %q. Expecting %q", path, ext, expectedExt) - } -} - -func TestServeFileContentType(t *testing.T) { - // This test can't run parallel as files in / might by changed by other tests. - - var ctx RequestCtx - var req Request - req.Header.SetMethod(MethodGet) - req.SetRequestURI("http://foobar.com/baz") - ctx.Init(&req, nil, nil) - - ServeFile(&ctx, "testdata/test.png") - - var resp Response - s := ctx.Response.String() - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - expected := []byte("image/png") - if !bytes.Equal(resp.Header.ContentType(), expected) { - t.Fatalf("Unexpected Content-Type, expected: %q got %q", expected, resp.Header.ContentType()) - } -} - -func TestServeFileDirectoryRedirect(t *testing.T) { - t.Parallel() - - if runtime.GOOS == "windows" { - t.SkipNow() - } - - var ctx RequestCtx - var req Request - req.SetRequestURI("http://foobar.com") - ctx.Init(&req, nil, nil) - - ctx.Request.Reset() - ctx.Response.Reset() - ServeFile(&ctx, "fasthttputil") - if ctx.Response.StatusCode() != StatusFound { - t.Fatalf("Unexpected status code %d for directory '/fasthttputil' without trailing slash. Expecting %d.", ctx.Response.StatusCode(), StatusFound) - } - - ctx.Request.Reset() - ctx.Response.Reset() - ServeFile(&ctx, "fasthttputil/") - if ctx.Response.StatusCode() != StatusOK { - t.Fatalf("Unexpected status code %d for directory '/fasthttputil/' with trailing slash. Expecting %d.", ctx.Response.StatusCode(), StatusOK) - } - - ctx.Request.Reset() - ctx.Response.Reset() - ServeFile(&ctx, "fs.go") - if ctx.Response.StatusCode() != StatusOK { - t.Fatalf("Unexpected status code %d for file '/fs.go'. Expecting %d.", ctx.Response.StatusCode(), StatusOK) - } -} diff --git a/lib/fasthttp/header_regression_test.go b/lib/fasthttp/header_regression_test.go deleted file mode 100644 index e19063759..000000000 --- a/lib/fasthttp/header_regression_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "fmt" - "os" - "strings" - "testing" -) - -func TestIssue28ResponseWithoutBodyNoContentType(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var r Response - - // Empty response without content-type - s := r.String() - if strings.Contains(s, "Content-Type") { - t.Fatalf("unexpected Content-Type found in response header with empty body: %q", s) - } - - // Explicitly set content-type - r.Header.SetContentType("foo/bar") - s = r.String() - if !strings.Contains(s, "Content-Type: foo/bar\r\n") { - t.Fatalf("missing explicitly set content-type for empty response: %q", s) - } - - // Non-empty response. - r.Reset() - r.SetBodyString("foobar") - s = r.String() - if !strings.Contains(s, fmt.Sprintf("Content-Type: %s\r\n", defaultContentType)) { - t.Fatalf("missing default content-type for non-empty response: %q", s) - } - - // Non-empty response with custom content-type. - r.Header.SetContentType("aaa/bbb") - s = r.String() - if !strings.Contains(s, "Content-Type: aaa/bbb\r\n") { - t.Fatalf("missing custom content-type: %q", s) - } -} - -func TestIssue6RequestHeaderSetContentType(t *testing.T) { - t.Parallel() - - testIssue6RequestHeaderSetContentType(t, MethodGet) - testIssue6RequestHeaderSetContentType(t, MethodPost) - testIssue6RequestHeaderSetContentType(t, MethodPut) - testIssue6RequestHeaderSetContentType(t, MethodPatch) -} - -func testIssue6RequestHeaderSetContentType(t *testing.T, method string) { - contentType := "application/json" - contentLength := 123 - - var h RequestHeader - h.SetMethod(method) - h.SetRequestURI("http://localhost/test") - h.SetContentType(contentType) - h.SetContentLength(contentLength) - - issue6VerifyRequestHeader(t, &h, contentType, contentLength, method) - - s := h.String() - - var h1 RequestHeader - - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - issue6VerifyRequestHeader(t, &h1, contentType, contentLength, method) -} - -func issue6VerifyRequestHeader(t *testing.T, h *RequestHeader, contentType string, contentLength int, method string) { - if string(h.ContentType()) != contentType { - t.Fatalf("unexpected content-type: %q. Expecting %q. method=%q", h.ContentType(), contentType, method) - } - if string(h.Method()) != method { - t.Fatalf("unexpected method: %q. Expecting %q", h.Method(), method) - } - if h.ContentLength() != contentLength { - t.Fatalf("unexpected content-length: %d. Expecting %d. method=%q", h.ContentLength(), contentLength, method) - } -} diff --git a/lib/fasthttp/header_test.go b/lib/fasthttp/header_test.go deleted file mode 100644 index f0be324df..000000000 --- a/lib/fasthttp/header_test.go +++ /dev/null @@ -1,2885 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "encoding/base64" - "errors" - "fmt" - "io" - "net/http" - "os" - "reflect" - "strings" - "testing" -) - -func TestResponseHeaderAddContentType(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var h ResponseHeader - h.Add("Content-Type", "test") - - got := string(h.Peek("Content-Type")) - expected := "test" - if got != expected { - t.Errorf("expected %q got %q", expected, got) - } - - var buf bytes.Buffer - h.WriteTo(&buf) //nolint:errcheck - - if n := strings.Count(buf.String(), "Content-Type: "); n != 1 { - t.Errorf("Content-Type occurred %d times", n) - } -} - -func TestResponseHeaderAddContentEncoding(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var h ResponseHeader - h.Add("Content-Encoding", "test") - - got := string(h.Peek("Content-Encoding")) - expected := "test" - if got != expected { - t.Errorf("expected %q got %q", expected, got) - } - - var buf bytes.Buffer - h.WriteTo(&buf) //nolint:errcheck - - if n := strings.Count(buf.String(), "Content-Encoding: "); n != 1 { - t.Errorf("Content-Encoding occurred %d times", n) - } -} - -func TestResponseHeaderMultiLineValue(t *testing.T) { - t.Parallel() - - s := "HTTP/1.1 200 SuperOK\r\n" + - "EmptyValue1:\r\n" + - "Content-Type: foo/bar;\r\n\tnewline;\r\n another/newline\r\n" + - "Foo: Bar\r\n" + - "Multi-Line: one;\r\n two\r\n" + - "Values: v1;\r\n v2; v3;\r\n v4;\tv5\r\n" + - "\r\n" - header := new(ResponseHeader) - if _, err := header.parse([]byte(s)); err != nil { - t.Fatalf("parse headers with multi-line values failed, %v", err) - } - response, err := http.ReadResponse(bufio.NewReader(strings.NewReader(s)), nil) - if err != nil { - t.Fatalf("parse response using net/http failed, %v", err) - } - - if !bytes.Equal(header.StatusMessage(), []byte("SuperOK")) { - t.Errorf("parse status line with non-default value failed, got: '%q' want: 'SuperOK'", header.StatusMessage()) - } - - header.SetProtocol([]byte("HTTP/3.3")) - if !bytes.Equal(header.Protocol(), []byte("HTTP/3.3")) { - t.Errorf("parse protocol with non-default value failed, got: '%q' want: 'HTTP/3.3'", header.Protocol()) - } - - if !bytes.Equal(header.appendStatusLine(nil), []byte("HTTP/3.3 200 SuperOK\r\n")) { - t.Errorf("parse status line with non-default value failed, got: '%q' want: 'HTTP/3.3 200 SuperOK'", header.Protocol()) - } - - header.SetStatusMessage(nil) - - if !bytes.Equal(header.appendStatusLine(nil), []byte("HTTP/3.3 200 OK\r\n")) { - t.Errorf("parse status line with default protocol value failed, got: '%q' want: 'HTTP/3.3 200 OK'", header.appendStatusLine(nil)) - } - - header.SetStatusMessage(s2b(StatusMessage(200))) - - if !bytes.Equal(header.appendStatusLine(nil), []byte("HTTP/3.3 200 OK\r\n")) { - t.Errorf("parse status line with default protocol value failed, got: '%q' want: 'HTTP/3.3 200 OK'", header.appendStatusLine(nil)) - } - - for name, vals := range response.Header { - got := string(header.Peek(name)) - want := vals[0] - - if got != want { - t.Errorf("unexpected %q got: %q want: %q", name, got, want) - } - } -} - -func TestResponseHeaderMultiLineName(t *testing.T) { - t.Parallel() - - s := "HTTP/1.1 200 OK\r\n" + - "Host: golang.org\r\n" + - "Gopher-New-\r\n" + - " Line: This is a header on multiple lines\r\n" + - "\r\n" - header := new(ResponseHeader) - if _, err := header.parse([]byte(s)); err != errInvalidName { - m := make(map[string]string) - header.VisitAll(func(key, value []byte) { - m[string(key)] = string(value) - }) - t.Errorf("expected error, got %q (%v)", m, err) - } - - if !bytes.Equal(header.StatusMessage(), []byte("OK")) { - t.Errorf("expected default status line, got: %q", header.StatusMessage()) - } - - if !bytes.Equal(header.Protocol(), []byte("HTTP/1.1")) { - t.Errorf("expected default protocol, got: %q", header.Protocol()) - } - - if !bytes.Equal(header.appendStatusLine(nil), []byte("HTTP/1.1 200 OK\r\n")) { - t.Errorf("parse status line with non-default value failed, got: %q want: HTTP/1.1 200 OK", header.Protocol()) - } -} - -func TestResponseHeaderMultiLinePaniced(t *testing.T) { - t.Parallel() - - // Input generated by fuzz testing that caused the parser to panic. - s, _ := base64.StdEncoding.DecodeString("aAEAIDoKKDoKICA6CgkKCiA6CiA6CgkpCiA6CiA6CiA6Cig6CiAgOgoJCgogOgogOgoJKQogOgogOgogOgogOgogOgoJOg86CiA6CiA6Cig6CiAyCg==") - header := new(RequestHeader) - header.parse(s) //nolint:errcheck -} - -func TestResponseHeaderEmptyValueFromHeader(t *testing.T) { - t.Parallel() - - var h1 ResponseHeader - h1.SetContentType("foo/bar") - h1.Set("EmptyValue1", "") - h1.Set("EmptyValue2", " ") - s := h1.String() - - var h ResponseHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(h.ContentType()) != string(h1.ContentType()) { - t.Fatalf("unexpected content-type: %q. Expecting %q", h.ContentType(), h1.ContentType()) - } - v1 := h.Peek("EmptyValue1") - if len(v1) > 0 { - t.Fatalf("expecting empty value. Got %q", v1) - } - v2 := h.Peek("EmptyValue2") - if len(v2) > 0 { - t.Fatalf("expecting empty value. Got %q", v2) - } -} - -func TestResponseHeaderEmptyValueFromString(t *testing.T) { - t.Parallel() - - s := "HTTP/1.1 200 OK\r\n" + - "EmptyValue1:\r\n" + - "Content-Type: foo/bar\r\n" + - "EmptyValue2: \r\n" + - "\r\n" - - var h ResponseHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(h.ContentType()) != "foo/bar" { - t.Fatalf("unexpected content-type: %q. Expecting %q", h.ContentType(), "foo/bar") - } - v1 := h.Peek("EmptyValue1") - if len(v1) > 0 { - t.Fatalf("expecting empty value. Got %q", v1) - } - v2 := h.Peek("EmptyValue2") - if len(v2) > 0 { - t.Fatalf("expecting empty value. Got %q", v2) - } -} - -func TestRequestHeaderEmptyValueFromHeader(t *testing.T) { - t.Parallel() - - var h1 RequestHeader - h1.SetRequestURI("/foo/bar") - h1.SetHost("foobar") - h1.Set("EmptyValue1", "") - h1.Set("EmptyValue2", " ") - s := h1.String() - - var h RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(h.Host()) != string(h1.Host()) { - t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), h1.Host()) - } - v1 := h.Peek("EmptyValue1") - if len(v1) > 0 { - t.Fatalf("expecting empty value. Got %q", v1) - } - v2 := h.Peek("EmptyValue2") - if len(v2) > 0 { - t.Fatalf("expecting empty value. Got %q", v2) - } -} - -func TestRequestHeaderEmptyValueFromString(t *testing.T) { - t.Parallel() - - s := "GET / HTTP/1.1\r\n" + - "EmptyValue1:\r\n" + - "Host: foobar\r\n" + - "EmptyValue2: \r\n" + - "\r\n" - var h RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(h.Host()) != "foobar" { - t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), "foobar") - } - v1 := h.Peek("EmptyValue1") - if len(v1) > 0 { - t.Fatalf("expecting empty value. Got %q", v1) - } - v2 := h.Peek("EmptyValue2") - if len(v2) > 0 { - t.Fatalf("expecting empty value. Got %q", v2) - } -} - -func TestRequestRawHeaders(t *testing.T) { - t.Parallel() - - kvs := "hOsT: foobar\r\n" + - "value: b\r\n" + - "\r\n" - t.Run("normalized", func(t *testing.T) { - s := "GET / HTTP/1.1\r\n" + kvs - exp := kvs - var h RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(h.Host()) != "foobar" { - t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), "foobar") - } - v2 := h.Peek("Value") - if !bytes.Equal(v2, []byte{'b'}) { - t.Fatalf("expecting non empty value. Got %q", v2) - } - if raw := h.RawHeaders(); string(raw) != exp { - t.Fatalf("expected header %q, got %q", exp, raw) - } - }) - for _, n := range []int{0, 1, 4, 8} { - t.Run(fmt.Sprintf("post-%dk", n), func(t *testing.T) { - l := 1024 * n - body := make([]byte, l) - for i := range body { - body[i] = 'a' - } - cl := fmt.Sprintf("Content-Length: %d\r\n", l) - s := "POST / HTTP/1.1\r\n" + cl + kvs + string(body) - exp := cl + kvs - var h RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(h.Host()) != "foobar" { - t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), "foobar") - } - v2 := h.Peek("Value") - if !bytes.Equal(v2, []byte{'b'}) { - t.Fatalf("expecting non empty value. Got %q", v2) - } - if raw := h.RawHeaders(); string(raw) != exp { - t.Fatalf("expected header %q, got %q", exp, raw) - } - }) - } - t.Run("http10", func(t *testing.T) { - s := "GET / HTTP/1.0\r\n" + kvs - exp := kvs - var h RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(h.Host()) != "foobar" { - t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), "foobar") - } - v2 := h.Peek("Value") - if !bytes.Equal(v2, []byte{'b'}) { - t.Fatalf("expecting non empty value. Got %q", v2) - } - if raw := h.RawHeaders(); string(raw) != exp { - t.Fatalf("expected header %q, got %q", exp, raw) - } - }) - t.Run("no-kvs", func(t *testing.T) { - s := "GET / HTTP/1.1\r\n\r\n" - exp := "" - var h RequestHeader - h.DisableNormalizing() - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(h.Host()) != "" { - t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), "") - } - v1 := h.Peek("NoKey") - if len(v1) > 0 { - t.Fatalf("expecting empty value. Got %q", v1) - } - if raw := h.RawHeaders(); string(raw) != exp { - t.Fatalf("expected header %q, got %q", exp, raw) - } - }) -} - -func TestRequestHeaderSetCookieWithSpecialChars(t *testing.T) { - t.Parallel() - - var h RequestHeader - h.Set("Cookie", "ID&14") - s := h.String() - - if !strings.Contains(s, "Cookie: ID&14") { - t.Fatalf("Missing cookie in request header: %q", s) - } - - var h1 RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - cookie := h1.Peek(HeaderCookie) - if string(cookie) != "ID&14" { - t.Fatalf("unexpected cooke: %q. Expecting %q", cookie, "ID&14") - } - - cookie = h1.Cookie("") - if string(cookie) != "ID&14" { - t.Fatalf("unexpected cooke: %q. Expecting %q", cookie, "ID&14") - } -} - -func TestResponseHeaderDefaultStatusCode(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var h ResponseHeader - statusCode := h.StatusCode() - if statusCode != StatusOK { - t.Fatalf("unexpected status code: %d. Expecting %d", statusCode, StatusOK) - } -} - -func TestResponseHeaderDelClientCookie(t *testing.T) { - t.Parallel() - - cookieName := "foobar" - - var h ResponseHeader - c := AcquireCookie() - c.SetKey(cookieName) - c.SetValue("aasdfsdaf") - h.SetCookie(c) - - h.DelClientCookieBytes([]byte(cookieName)) - if !h.Cookie(c) { - t.Fatalf("expecting cookie %q", c.Key()) - } - if !c.Expire().Equal(CookieExpireDelete) { - t.Fatalf("unexpected cookie expiration time: %q. Expecting %q", c.Expire(), CookieExpireDelete) - } - if len(c.Value()) > 0 { - t.Fatalf("unexpected cookie value: %q. Expecting empty value", c.Value()) - } - ReleaseCookie(c) -} - -func TestResponseHeaderAdd(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - m := make(map[string]struct{}) - var h ResponseHeader - h.Add("aaa", "bbb") - h.Add("content-type", "xxx") - m["bbb"] = struct{}{} - m["xxx"] = struct{}{} - for i := 0; i < 10; i++ { - v := fmt.Sprintf("%d", i) - h.Add("Foo-Bar", v) - m[v] = struct{}{} - } - if h.Len() != 12 { - t.Fatalf("unexpected header len %d. Expecting 12", h.Len()) - } - - h.VisitAll(func(k, v []byte) { - switch string(k) { - case "Aaa", "Foo-Bar", "Content-Type": - if _, ok := m[string(v)]; !ok { - t.Fatalf("unexpected value found %q. key %q", v, k) - } - delete(m, string(v)) - default: - t.Fatalf("unexpected key found: %q", k) - } - }) - if len(m) > 0 { - t.Fatalf("%d headers are missed", len(m)) - } - - s := h.String() - br := bufio.NewReader(bytes.NewBufferString(s)) - var h1 ResponseHeader - if err := h1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - h.VisitAll(func(k, v []byte) { - switch string(k) { - case "Aaa", "Foo-Bar", "Content-Type": - m[string(v)] = struct{}{} - default: - t.Fatalf("unexpected key found: %q", k) - } - }) - if len(m) != 12 { - t.Fatalf("unexpected number of headers: %d. Expecting 12", len(m)) - } -} - -func TestRequestHeaderAdd(t *testing.T) { - t.Parallel() - - m := make(map[string]struct{}) - var h RequestHeader - h.Add("aaa", "bbb") - h.Add("user-agent", "xxx") - m["bbb"] = struct{}{} - m["xxx"] = struct{}{} - for i := 0; i < 10; i++ { - v := fmt.Sprintf("%d", i) - h.Add("Foo-Bar", v) - m[v] = struct{}{} - } - if h.Len() != 12 { - t.Fatalf("unexpected header len %d. Expecting 12", h.Len()) - } - - h.VisitAll(func(k, v []byte) { - switch string(k) { - case "Aaa", "Foo-Bar", "User-Agent": - if _, ok := m[string(v)]; !ok { - t.Fatalf("unexpected value found %q. key %q", v, k) - } - delete(m, string(v)) - default: - t.Fatalf("unexpected key found: %q", k) - } - }) - if len(m) > 0 { - t.Fatalf("%d headers are missed", len(m)) - } - - s := h.String() - br := bufio.NewReader(bytes.NewBufferString(s)) - var h1 RequestHeader - if err := h1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - h.VisitAll(func(k, v []byte) { - switch string(k) { - case "Aaa", "Foo-Bar", "User-Agent": - m[string(v)] = struct{}{} - default: - t.Fatalf("unexpected key found: %q", k) - } - }) - if len(m) != 12 { - t.Fatalf("unexpected number of headers: %d. Expecting 12", len(m)) - } - s1 := h1.String() - if s != s1 { - t.Fatalf("unexpected headers %q. Expecting %q", s1, s) - } -} - -func TestHasHeaderValue(t *testing.T) { - t.Parallel() - - testHasHeaderValue(t, "foobar", "foobar", true) - testHasHeaderValue(t, "foobar", "foo", false) - testHasHeaderValue(t, "foobar", "bar", false) - testHasHeaderValue(t, "keep-alive, Upgrade", "keep-alive", true) - testHasHeaderValue(t, "keep-alive , Upgrade", "Upgrade", true) - testHasHeaderValue(t, "keep-alive, Upgrade", "Upgrade-foo", false) - testHasHeaderValue(t, "keep-alive, Upgrade", "Upgr", false) - testHasHeaderValue(t, "foo , bar, baz ,", "foo", true) - testHasHeaderValue(t, "foo , bar, baz ,", "bar", true) - testHasHeaderValue(t, "foo , bar, baz ,", "baz", true) - testHasHeaderValue(t, "foo , bar, baz ,", "ba", false) - testHasHeaderValue(t, "foo, ", "", true) - testHasHeaderValue(t, "foo", "", false) -} - -func testHasHeaderValue(t *testing.T, s, value string, has bool) { - ok := hasHeaderValue([]byte(s), []byte(value)) - if ok != has { - t.Fatalf("unexpected hasHeaderValue(%q, %q)=%v. Expecting %v", s, value, ok, has) - } -} - -func TestRequestHeaderDel(t *testing.T) { - t.Parallel() - - var h RequestHeader - h.Set("Foo-Bar", "baz") - h.Set("aaa", "bbb") - h.Set(HeaderConnection, "keep-alive") - h.Set("Content-Type", "aaa") - h.Set(HeaderHost, "aaabbb") - h.Set("User-Agent", "asdfas") - h.Set("Content-Length", "1123") - h.Set("Cookie", "foobar=baz") - h.Set(HeaderTrailer, "foo, bar") - - h.Del("foo-bar") - h.Del("connection") - h.DelBytes([]byte("content-type")) - h.Del("Host") - h.Del("user-agent") - h.Del("content-length") - h.Del("cookie") - h.Del("trailer") - - hv := h.Peek("aaa") - if string(hv) != "bbb" { - t.Fatalf("unexpected header value: %q. Expecting %q", hv, "bbb") - } - hv = h.Peek("Foo-Bar") - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderConnection) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderContentType) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderHost) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderUserAgent) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderContentLength) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderCookie) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderTrailer) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - - cv := h.Cookie("foobar") - if len(cv) > 0 { - t.Fatalf("unexpected cookie obtianed: %q", cv) - } - if h.ContentLength() != 0 { - t.Fatalf("unexpected content-length: %d. Expecting 0", h.ContentLength()) - } -} - -func TestResponseHeaderDel(t *testing.T) { - t.Parallel() - - var h ResponseHeader - h.Set("Foo-Bar", "baz") - h.Set("aaa", "bbb") - h.Set(HeaderConnection, "keep-alive") - h.Set(HeaderContentType, "aaa") - h.Set(HeaderContentEncoding, "gzip") - h.Set(HeaderServer, "aaabbb") - h.Set(HeaderContentLength, "1123") - h.Set(HeaderTrailer, "foo, bar") - - var c Cookie - c.SetKey("foo") - c.SetValue("bar") - h.SetCookie(&c) - - h.Del("foo-bar") - h.Del("connection") - h.DelBytes([]byte("content-type")) - h.Del(HeaderServer) - h.Del("content-length") - h.Del("set-cookie") - h.Del("trailer") - - hv := h.Peek("aaa") - if string(hv) != "bbb" { - t.Fatalf("unexpected header value: %q. Expecting %q", hv, "bbb") - } - hv = h.Peek("Foo-Bar") - if len(hv) > 0 { - t.Fatalf("non-zero header value: %q", hv) - } - hv = h.Peek(HeaderConnection) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderContentType) - if string(hv) != string(defaultContentType) { - t.Fatalf("unexpected content-type: %q. Expecting %q", hv, defaultContentType) - } - hv = h.Peek(HeaderContentEncoding) - if string(hv) != ("gzip") { - t.Fatalf("unexpected content-encoding: %q. Expecting %q", hv, "gzip") - } - hv = h.Peek(HeaderServer) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderContentLength) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - hv = h.Peek(HeaderTrailer) - if len(hv) > 0 { - t.Fatalf("non-zero value: %q", hv) - } - - if h.Cookie(&c) { - t.Fatalf("unexpected cookie obtianed: %q", &c) - } - if h.ContentLength() != 0 { - t.Fatalf("unexpected content-length: %d. Expecting 0", h.ContentLength()) - } -} - -func TestResponseHeaderSetTrailerGetBytes(t *testing.T) { - t.Parallel() - - h := &ResponseHeader{} - h.noDefaultDate = true - h.Set("Foo", "bar") - h.Set(HeaderTrailer, "Baz") - h.Set("Baz", "test") - - headerBytes := h.Header() - n, err := h.parseFirstLine(headerBytes) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if string(headerBytes[n:]) != "Foo: bar\r\nTrailer: Baz\r\n\r\n" { - t.Fatalf("Unexpected header: %q. Expected %q", headerBytes[n:], "Foo: bar\nTrailer: Baz\n\n") - } - if string(h.TrailerHeader()) != "Baz: test\r\n\r\n" { - t.Fatalf("Unexpected trailer header: %q. Expected %q", h.TrailerHeader(), "Baz: test\r\n\r\n") - } -} - -func TestRequestHeaderSetTrailerGetBytes(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - h := &RequestHeader{} - h.Set("Foo", "bar") - h.Set(HeaderTrailer, "Baz") - h.Set("Baz", "test") - - headerBytes := h.Header() - n, err := h.parseFirstLine(headerBytes) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if string(headerBytes[n:]) != "Foo: bar\r\nTrailer: Baz\r\n\r\n" { - t.Fatalf("Unexpected header: %q. Expected %q", headerBytes[n:], "Foo: bar\nTrailer: Baz\n\n") - } - if string(h.TrailerHeader()) != "Baz: test\r\n\r\n" { - t.Fatalf("Unexpected trailer header: %q. Expected %q", h.TrailerHeader(), "Baz: test\r\n\r\n") - } -} - -func TestAppendNormalizedHeaderKeyBytes(t *testing.T) { - t.Parallel() - - testAppendNormalizedHeaderKeyBytes(t, "", "") - testAppendNormalizedHeaderKeyBytes(t, "Content-Type", "Content-Type") - testAppendNormalizedHeaderKeyBytes(t, "foO-bAr-BAZ", "Foo-Bar-Baz") -} - -func testAppendNormalizedHeaderKeyBytes(t *testing.T, key, expectedKey string) { - buf := []byte("foobar") - result := AppendNormalizedHeaderKeyBytes(buf, []byte(key)) - normalizedKey := result[len(buf):] - if string(normalizedKey) != expectedKey { - t.Fatalf("unexpected normalized key %q. Expecting %q", normalizedKey, expectedKey) - } -} - -func TestRequestHeaderHTTP10ConnectionClose(t *testing.T) { - t.Parallel() - - s := "GET / HTTP/1.0\r\nHost: foobar\r\n\r\n" - var h RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if !h.ConnectionClose() { - t.Fatalf("expecting 'Connection: close' request header") - } -} - -func TestRequestHeaderHTTP10ConnectionKeepAlive(t *testing.T) { - t.Parallel() - - s := "GET / HTTP/1.0\r\nHost: foobar\r\nConnection: keep-alive\r\n\r\n" - var h RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if h.ConnectionClose() { - t.Fatalf("unexpected 'Connection: close' request header") - } -} - -func TestBufferSnippet(t *testing.T) { - t.Parallel() - - testBufferSnippet(t, "", `""`) - testBufferSnippet(t, "foobar", `"foobar"`) - - b := string(createFixedBody(199)) - bExpected := fmt.Sprintf("%q", b) - testBufferSnippet(t, b, bExpected) - for i := 0; i < 10; i++ { - b += "foobar" - bExpected = fmt.Sprintf("%q", b) - testBufferSnippet(t, b, bExpected) - } - - b = string(createFixedBody(400)) - bExpected = fmt.Sprintf("%q", b) - testBufferSnippet(t, b, bExpected) - for i := 0; i < 10; i++ { - b += "sadfqwer" - bExpected = fmt.Sprintf("%q...%q", b[:200], b[len(b)-200:]) - testBufferSnippet(t, b, bExpected) - } -} - -func testBufferSnippet(t *testing.T, buf, expectedSnippet string) { - snippet := bufferSnippet([]byte(buf)) - if snippet != expectedSnippet { - t.Fatalf("unexpected snippet %q. Expecting %q", snippet, expectedSnippet) - } -} - -func TestResponseHeaderTrailingCRLFSuccess(t *testing.T) { - t.Parallel() - - trailingCRLF := "\r\n\r\n\r\n" - s := "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nContent-Length: 123\r\n\r\n" + trailingCRLF - - var r ResponseHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // try reading the trailing CRLF. It must return EOF - err := r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err != io.EOF { - t.Fatalf("unexpected error: %v. Expecting %v", err, io.EOF) - } -} - -func TestResponseHeaderTrailingCRLFError(t *testing.T) { - t.Parallel() - - trailingCRLF := "\r\nerror\r\n\r\n" - s := "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nContent-Length: 123\r\n\r\n" + trailingCRLF - - var r ResponseHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // try reading the trailing CRLF. It must return EOF - err := r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err == io.EOF { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestRequestHeaderTrailingCRLFSuccess(t *testing.T) { - t.Parallel() - - trailingCRLF := "\r\n\r\n\r\n" - s := "GET / HTTP/1.1\r\nHost: aaa.com\r\n\r\n" + trailingCRLF - - var r RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // try reading the trailing CRLF. It must return EOF - err := r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err != io.EOF { - t.Fatalf("unexpected error: %v. Expecting %v", err, io.EOF) - } -} - -func TestRequestHeaderTrailingCRLFError(t *testing.T) { - t.Parallel() - - trailingCRLF := "\r\nerror\r\n\r\n" - s := "GET / HTTP/1.1\r\nHost: aaa.com\r\n\r\n" + trailingCRLF - - var r RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // try reading the trailing CRLF. It must return EOF - err := r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err == io.EOF { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestRequestHeaderReadEOF(t *testing.T) { - t.Parallel() - - var r RequestHeader - - br := bufio.NewReader(&bytes.Buffer{}) - err := r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err != io.EOF { - t.Fatalf("unexpected error: %v. Expecting %v", err, io.EOF) - } - - // incomplete request header mustn't return io.EOF - br = bufio.NewReader(bytes.NewBufferString("GET ")) - err = r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err == io.EOF { - t.Fatalf("expecting non-EOF error") - } -} - -func TestResponseHeaderReadEOF(t *testing.T) { - t.Parallel() - - var r ResponseHeader - - br := bufio.NewReader(&bytes.Buffer{}) - err := r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err != io.EOF { - t.Fatalf("unexpected error: %v. Expecting %v", err, io.EOF) - } - - // incomplete response header mustn't return io.EOF - br = bufio.NewReader(bytes.NewBufferString("HTTP/1.1 ")) - err = r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err == io.EOF { - t.Fatalf("expecting non-EOF error") - } -} - -func TestResponseHeaderOldVersion(t *testing.T) { - t.Parallel() - - var h ResponseHeader - - s := "HTTP/1.0 200 OK\r\nContent-Length: 5\r\nContent-Type: aaa\r\n\r\n12345" - s += "HTTP/1.0 200 OK\r\nContent-Length: 2\r\nContent-Type: ass\r\nConnection: keep-alive\r\n\r\n42" - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !h.ConnectionClose() { - t.Fatalf("expecting 'Connection: close' for the response with old http protocol") - } - - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if h.ConnectionClose() { - t.Fatalf("unexpected 'Connection: close' for keep-alive response with old http protocol") - } -} - -func TestRequestHeaderSetByteRange(t *testing.T) { - t.Parallel() - - testRequestHeaderSetByteRange(t, 0, 10, "bytes=0-10") - testRequestHeaderSetByteRange(t, 123, -1, "bytes=123-") - testRequestHeaderSetByteRange(t, -234, 58349, "bytes=-234") -} - -func testRequestHeaderSetByteRange(t *testing.T, startPos, endPos int, expectedV string) { - var h RequestHeader - h.SetByteRange(startPos, endPos) - v := h.Peek(HeaderRange) - if string(v) != expectedV { - t.Fatalf("unexpected range: %q. Expecting %q. startPos=%d, endPos=%d", v, expectedV, startPos, endPos) - } -} - -func TestResponseHeaderSetContentRange(t *testing.T) { - t.Parallel() - - testResponseHeaderSetContentRange(t, 0, 0, 1, "bytes 0-0/1") - testResponseHeaderSetContentRange(t, 123, 456, 789, "bytes 123-456/789") -} - -func testResponseHeaderSetContentRange(t *testing.T, startPos, endPos, contentLength int, expectedV string) { - var h ResponseHeader - h.SetContentRange(startPos, endPos, contentLength) - v := h.Peek(HeaderContentRange) - if string(v) != expectedV { - t.Fatalf("unexpected content-range: %q. Expecting %q. startPos=%d, endPos=%d, contentLength=%d", - v, expectedV, startPos, endPos, contentLength) - } -} - -func TestRequestHeaderHasAcceptEncoding(t *testing.T) { - t.Parallel() - - testRequestHeaderHasAcceptEncoding(t, "", "gzip", false) - testRequestHeaderHasAcceptEncoding(t, "gzip", "sdhc", false) - testRequestHeaderHasAcceptEncoding(t, "deflate", "deflate", true) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "gzi", false) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "dhc", false) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "sdh", false) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "zip", false) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "flat", false) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "flate", false) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "def", false) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "gzip", true) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "deflate", true) - testRequestHeaderHasAcceptEncoding(t, "gzip, deflate, sdhc", "sdhc", true) -} - -func testRequestHeaderHasAcceptEncoding(t *testing.T, ae, v string, resultExpected bool) { - var h RequestHeader - h.Set(HeaderAcceptEncoding, ae) - result := h.HasAcceptEncoding(v) - if result != resultExpected { - t.Fatalf("unexpected result in HasAcceptEncoding(%q, %q): %v. Expecting %v", ae, v, result, resultExpected) - } -} - -func TestRequestMultipartFormBoundary(t *testing.T) { - t.Parallel() - - testRequestMultipartFormBoundary(t, "POST / HTTP/1.1\r\nContent-Type: multipart/form-data; boundary=foobar\r\n\r\n", "foobar") - - // incorrect content-type - testRequestMultipartFormBoundary(t, "POST / HTTP/1.1\r\nContent-Type: foo/bar\r\n\r\n", "") - - // empty boundary - testRequestMultipartFormBoundary(t, "POST / HTTP/1.1\r\nContent-Type: multipart/form-data; boundary=\r\n\r\n", "") - - // missing boundary - testRequestMultipartFormBoundary(t, "POST / HTTP/1.1\r\nContent-Type: multipart/form-data\r\n\r\n", "") - - // boundary after other content-type params - testRequestMultipartFormBoundary(t, "POST / HTTP/1.1\r\nContent-Type: multipart/form-data; foo=bar; boundary=--aaabb \r\n\r\n", "--aaabb") - - // quoted boundary - testRequestMultipartFormBoundary(t, "POST / HTTP/1.1\r\nContent-Type: multipart/form-data; boundary=\"foobar\"\r\n\r\n", "foobar") - - var h RequestHeader - h.SetMultipartFormBoundary("foobarbaz") - b := h.MultipartFormBoundary() - if string(b) != "foobarbaz" { - t.Fatalf("unexpected boundary %q. Expecting %q", b, "foobarbaz") - } -} - -func testRequestMultipartFormBoundary(t *testing.T, s, boundary string) { - var h RequestHeader - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v. s=%q, boundary=%q", err, s, boundary) - } - - b := h.MultipartFormBoundary() - if string(b) != boundary { - t.Fatalf("unexpected boundary %q. Expecting %q. s=%q", b, boundary, s) - } -} - -func TestResponseHeaderConnectionUpgrade(t *testing.T) { - t.Parallel() - - testResponseHeaderConnectionUpgrade(t, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: Upgrade, HTTP2-Settings\r\n\r\n", - true, true) - testResponseHeaderConnectionUpgrade(t, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: keep-alive, Upgrade\r\n\r\n", - true, true) - - // non-http/1.1 protocol has 'connection: close' by default, which also disables 'connection: upgrade' - testResponseHeaderConnectionUpgrade(t, "HTTP/1.0 200 OK\r\nContent-Length: 10\r\nConnection: Upgrade, HTTP2-Settings\r\n\r\n", - false, false) - - // explicit keep-alive for non-http/1.1, so 'connection: upgrade' works - testResponseHeaderConnectionUpgrade(t, "HTTP/1.0 200 OK\r\nContent-Length: 10\r\nConnection: Upgrade, keep-alive\r\n\r\n", - true, true) - - // implicit keep-alive for http/1.1 - testResponseHeaderConnectionUpgrade(t, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n", false, true) - - // no content-length, so 'connection: close' is assumed - testResponseHeaderConnectionUpgrade(t, "HTTP/1.1 200 OK\r\n\r\n", false, false) -} - -func testResponseHeaderConnectionUpgrade(t *testing.T, s string, isUpgrade, isKeepAlive bool) { - var h ResponseHeader - - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v. Response header %q", err, s) - } - upgrade := h.ConnectionUpgrade() - if upgrade != isUpgrade { - t.Fatalf("unexpected 'connection: upgrade' when parsing response header: %v. Expecting %v. header %q. v=%q", - upgrade, isUpgrade, s, h.Peek("Connection")) - } - keepAlive := !h.ConnectionClose() - if keepAlive != isKeepAlive { - t.Fatalf("unexpected 'connection: keep-alive' when parsing response header: %v. Expecting %v. header %q. v=%q", - keepAlive, isKeepAlive, s, &h) - } -} - -func TestRequestHeaderConnectionUpgrade(t *testing.T) { - t.Parallel() - - testRequestHeaderConnectionUpgrade(t, "GET /foobar HTTP/1.1\r\nConnection: Upgrade, HTTP2-Settings\r\nHost: foobar.com\r\n\r\n", - true, true) - testRequestHeaderConnectionUpgrade(t, "GET /foobar HTTP/1.1\r\nConnection: keep-alive,Upgrade\r\nHost: foobar.com\r\n\r\n", - true, true) - - // non-http/1.1 has 'connection: close' by default, which resets 'connection: upgrade' - testRequestHeaderConnectionUpgrade(t, "GET /foobar HTTP/1.0\r\nConnection: Upgrade, HTTP2-Settings\r\nHost: foobar.com\r\n\r\n", - false, false) - - // explicit 'connection: keep-alive' in non-http/1.1 - testRequestHeaderConnectionUpgrade(t, "GET /foobar HTTP/1.0\r\nConnection: foo, Upgrade, keep-alive\r\nHost: foobar.com\r\n\r\n", - true, true) - - // no upgrade - testRequestHeaderConnectionUpgrade(t, "GET /foobar HTTP/1.1\r\nConnection: Upgradess, foobar\r\nHost: foobar.com\r\n\r\n", - false, true) - testRequestHeaderConnectionUpgrade(t, "GET /foobar HTTP/1.1\r\nHost: foobar.com\r\n\r\n", - false, true) - - // explicit connection close - testRequestHeaderConnectionUpgrade(t, "GET /foobar HTTP/1.1\r\nConnection: close\r\nHost: foobar.com\r\n\r\n", - false, false) -} - -func testRequestHeaderConnectionUpgrade(t *testing.T, s string, isUpgrade, isKeepAlive bool) { - var h RequestHeader - - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v. Request header %q", err, s) - } - upgrade := h.ConnectionUpgrade() - if upgrade != isUpgrade { - t.Fatalf("unexpected 'connection: upgrade' when parsing request header: %v. Expecting %v. header %q", - upgrade, isUpgrade, s) - } - keepAlive := !h.ConnectionClose() - if keepAlive != isKeepAlive { - t.Fatalf("unexpected 'connection: keep-alive' when parsing request header: %v. Expecting %v. header %q", - keepAlive, isKeepAlive, s) - } -} - -func TestRequestHeaderProxyWithCookie(t *testing.T) { - t.Parallel() - - // Proxy request header (read it, then write it without touching any headers). - var h RequestHeader - r := bytes.NewBufferString("GET /foo HTTP/1.1\r\nFoo: bar\r\nHost: aaa.com\r\nCookie: foo=bar; bazzz=aaaaaaa; x=y\r\nCookie: aqqqqq=123\r\n\r\n") - br := bufio.NewReader(r) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := h.Write(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var h1 RequestHeader - br.Reset(w) - if err := h1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(h1.RequestURI()) != "/foo" { - t.Fatalf("unexpected requestURI: %q. Expecting %q", h1.RequestURI(), "/foo") - } - if string(h1.Host()) != "aaa.com" { - t.Fatalf("unexpected host: %q. Expecting %q", h1.Host(), "aaa.com") - } - if string(h1.Peek("Foo")) != "bar" { - t.Fatalf("unexpected Foo: %q. Expecting %q", h1.Peek("Foo"), "bar") - } - if string(h1.Cookie("foo")) != "bar" { - t.Fatalf("unexpected coookie foo=%q. Expecting %q", h1.Cookie("foo"), "bar") - } - if string(h1.Cookie("bazzz")) != "aaaaaaa" { - t.Fatalf("unexpected cookie bazzz=%q. Expecting %q", h1.Cookie("bazzz"), "aaaaaaa") - } - if string(h1.Cookie("x")) != "y" { - t.Fatalf("unexpected cookie x=%q. Expecting %q", h1.Cookie("x"), "y") - } - if string(h1.Cookie("aqqqqq")) != "123" { - t.Fatalf("unexpected cookie aqqqqq=%q. Expecting %q", h1.Cookie("aqqqqq"), "123") - } -} - -func TestResponseHeaderFirstByteReadEOF(t *testing.T) { - t.Parallel() - - var h ResponseHeader - - r := &errorReader{fmt.Errorf("non-eof error")} - br := bufio.NewReader(r) - err := h.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err != io.EOF { - t.Fatalf("unexpected error %v. Expecting %v", err, io.EOF) - } -} - -type errorReader struct { - err error -} - -func (r *errorReader) Read(p []byte) (int, error) { - return 0, r.err -} - -func TestRequestHeaderEmptyMethod(t *testing.T) { - t.Parallel() - - var h RequestHeader - - if !h.IsGet() { - t.Fatalf("empty method must be equivalent to GET") - } -} - -func TestResponseHeaderHTTPVer(t *testing.T) { - t.Parallel() - - // non-http/1.1 - testResponseHeaderHTTPVer(t, "HTTP/1.0 200 OK\r\nContent-Type: aaa\r\nContent-Length: 123\r\n\r\n", true) - testResponseHeaderHTTPVer(t, "HTTP/0.9 200 OK\r\nContent-Type: aaa\r\nContent-Length: 123\r\n\r\n", true) - testResponseHeaderHTTPVer(t, "foobar 200 OK\r\nContent-Type: aaa\r\nContent-Length: 123\r\n\r\n", true) - - // http/1.1 - testResponseHeaderHTTPVer(t, "HTTP/1.1 200 OK\r\nContent-Type: aaa\r\nContent-Length: 123\r\n\r\n", false) -} - -func TestRequestHeaderHTTPVer(t *testing.T) { - t.Parallel() - - // non-http/1.1 - testRequestHeaderHTTPVer(t, "GET / HTTP/1.0\r\nHost: aa.com\r\n\r\n", true) - testRequestHeaderHTTPVer(t, "GET / HTTP/0.9\r\nHost: aa.com\r\n\r\n", true) - testRequestHeaderHTTPVer(t, "GET / foobar\r\nHost: aa.com\r\n\r\n", true) - - // empty http version - testRequestHeaderHTTPVer(t, "GET /\r\nHost: aaa.com\r\n\r\n", true) - testRequestHeaderHTTPVer(t, "GET / \r\nHost: aaa.com\r\n\r\n", true) - - // http/1.1 - testRequestHeaderHTTPVer(t, "GET / HTTP/1.1\r\nHost: a.com\r\n\r\n", false) -} - -func testResponseHeaderHTTPVer(t *testing.T, s string, connectionClose bool) { - var h ResponseHeader - - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v. response=%q", err, s) - } - if h.ConnectionClose() != connectionClose { - t.Fatalf("unexpected connectionClose %v. Expecting %v. response=%q", h.ConnectionClose(), connectionClose, s) - } -} - -func testRequestHeaderHTTPVer(t *testing.T, s string, connectionClose bool) { - var h RequestHeader - - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - if err := h.Read(br); err != nil { - t.Fatalf("unexpected error: %v. request=%q", err, s) - } - if h.ConnectionClose() != connectionClose { - t.Fatalf("unexpected connectionClose %v. Expecting %v. request=%q", h.ConnectionClose(), connectionClose, s) - } -} - -func TestResponseHeaderCopyTo(t *testing.T) { - t.Parallel() - - var h ResponseHeader - - h.Set(HeaderSetCookie, "foo=bar") - h.Set(HeaderContentType, "foobar") - h.Set(HeaderContentEncoding, "gzip") - h.Set("AAA-BBB", "aaaa") - h.Set(HeaderTrailer, "foo, bar") - - var h1 ResponseHeader - h.CopyTo(&h1) - if !bytes.Equal(h1.Peek("Set-cookie"), h.Peek("Set-Cookie")) { - t.Fatalf("unexpected cookie %q. Expected %q", h1.Peek("set-cookie"), h.Peek("set-cookie")) - } - if !bytes.Equal(h1.Peek(HeaderContentType), h.Peek(HeaderContentType)) { - t.Fatalf("unexpected content-type %q. Expected %q", h1.Peek("content-type"), h.Peek("content-type")) - } - if !bytes.Equal(h1.Peek(HeaderContentEncoding), h.Peek(HeaderContentEncoding)) { - t.Fatalf("unexpected content-encoding %q. Expected %q", h1.Peek("content-encoding"), h.Peek("content-encoding")) - } - if !bytes.Equal(h1.Peek("aaa-bbb"), h.Peek("AAA-BBB")) { - t.Fatalf("unexpected aaa-bbb %q. Expected %q", h1.Peek("aaa-bbb"), h.Peek("aaa-bbb")) - } - if !bytes.Equal(h1.Peek(HeaderTrailer), h.Peek(HeaderTrailer)) { - t.Fatalf("unexpected trailer %q. Expected %q", h1.Peek(HeaderTrailer), h.Peek(HeaderTrailer)) - } - - // flush buf - h.bufKV = argsKV{} - h1.bufKV = argsKV{} - - if !reflect.DeepEqual(h, h1) { //nolint:govet - t.Fatalf("ResponseHeaderCopyTo fail, src: \n%+v\ndst: \n%+v\n", h, h1) //nolint:govet - } -} - -func TestRequestHeaderCopyTo(t *testing.T) { - t.Parallel() - - var h RequestHeader - - h.Set(HeaderCookie, "aa=bb; cc=dd") - h.Set(HeaderContentType, "foobar") - h.Set(HeaderContentEncoding, "gzip") - h.Set(HeaderHost, "aaaa") - h.Set("aaaxxx", "123") - h.Set(HeaderTrailer, "foo, bar") - - var h1 RequestHeader - h.CopyTo(&h1) - if !bytes.Equal(h1.Peek("cookie"), h.Peek(HeaderCookie)) { - t.Fatalf("unexpected cookie after copying: %q. Expected %q", h1.Peek("cookie"), h.Peek("cookie")) - } - if !bytes.Equal(h1.Peek("content-type"), h.Peek(HeaderContentType)) { - t.Fatalf("unexpected content-type %q. Expected %q", h1.Peek("content-type"), h.Peek("content-type")) - } - if !bytes.Equal(h1.Peek("content-encoding"), h.Peek(HeaderContentEncoding)) { - t.Fatalf("unexpected content-encoding %q. Expected %q", h1.Peek("content-encoding"), h.Peek("content-encoding")) - } - if !bytes.Equal(h1.Peek("host"), h.Peek("host")) { - t.Fatalf("unexpected host %q. Expected %q", h1.Peek("host"), h.Peek("host")) - } - if !bytes.Equal(h1.Peek("aaaxxx"), h.Peek("aaaxxx")) { - t.Fatalf("unexpected aaaxxx %q. Expected %q", h1.Peek("aaaxxx"), h.Peek("aaaxxx")) - } - if !bytes.Equal(h1.Peek(HeaderTrailer), h.Peek(HeaderTrailer)) { - t.Fatalf("unexpected trailer %q. Expected %q", h1.Peek(HeaderTrailer), h.Peek(HeaderTrailer)) - } - - // flush buf - h.bufKV = argsKV{} - h1.bufKV = argsKV{} - - if !reflect.DeepEqual(h, h1) { //nolint:govet - t.Fatalf("RequestHeaderCopyTo fail, src: \n%+v\ndst: \n%+v\n", h, h1) //nolint:govet - } -} - -func TestResponseContentTypeNoDefaultNotEmpty(t *testing.T) { - t.Parallel() - - var h ResponseHeader - - h.SetNoDefaultContentType(true) - h.SetContentLength(5) - - headers := h.String() - - if strings.Contains(headers, "Content-Type: \r\n") { - t.Fatalf("ResponseContentTypeNoDefaultNotEmpty fail, response: \n%+v\noutcome: \n%q\n", h, headers) //nolint:govet - } -} - -func TestRequestContentTypeDefaultNotEmpty(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var h RequestHeader - h.SetMethod(MethodPost) - h.SetContentLength(5) - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := h.Write(bw); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - var h1 RequestHeader - br := bufio.NewReader(w) - if err := h1.Read(br); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - if string(h1.contentType) != "application/octet-stream" { - t.Fatalf("unexpected Content-Type %q. Expecting %q", h1.contentType, "application/octet-stream") - } -} - -func TestRequestContentTypeNoDefault(t *testing.T) { - t.Parallel() - - var h RequestHeader - h.SetMethod(MethodDelete) - h.SetNoDefaultContentType(true) - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := h.Write(bw); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - var h1 RequestHeader - br := bufio.NewReader(w) - if err := h1.Read(br); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - if string(h1.contentType) != "" { - t.Fatalf("unexpected Content-Type %q. Expecting %q", h1.contentType, "") - } -} - -func TestResponseDateNoDefaultNotEmpty(t *testing.T) { - t.Parallel() - - var h ResponseHeader - - h.noDefaultDate = true - - headers := h.String() - - if strings.Contains(headers, "\r\nDate: ") { - t.Fatalf("ResponseDateNoDefaultNotEmpty fail, response: \n%+v\noutcome: \n%q\n", h, headers) //nolint:govet - } -} - -func TestRequestHeaderConnectionClose(t *testing.T) { - t.Parallel() - - var h RequestHeader - - h.Set(HeaderConnection, "close") - h.Set(HeaderHost, "foobar") - if !h.ConnectionClose() { - t.Fatalf("connection: close not set") - } - - var w bytes.Buffer - bw := bufio.NewWriter(&w) - if err := h.Write(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var h1 RequestHeader - br := bufio.NewReader(&w) - if err := h1.Read(br); err != nil { - t.Fatalf("error when reading request header: %v", err) - } - - if !h1.ConnectionClose() { - t.Fatalf("unexpected connection: close value: %v", h1.ConnectionClose()) - } - if string(h1.Peek(HeaderConnection)) != "close" { - t.Fatalf("unexpected connection value: %q. Expecting %q", h.Peek("Connection"), "close") - } - -} - -func TestRequestHeaderSetCookie(t *testing.T) { - t.Parallel() - - var h RequestHeader - - h.Set("Cookie", "foo=bar; baz=aaa") - h.Set("cOOkie", "xx=yyy") - - if string(h.Cookie("foo")) != "bar" { - t.Fatalf("Unexpected cookie %q. Expecting %q", h.Cookie("foo"), "bar") - } - if string(h.Cookie("baz")) != "aaa" { - t.Fatalf("Unexpected cookie %q. Expecting %q", h.Cookie("baz"), "aaa") - } - if string(h.Cookie("xx")) != "yyy" { - t.Fatalf("unexpected cookie %q. Expecting %q", h.Cookie("xx"), "yyy") - } -} - -func TestResponseHeaderSetCookie(t *testing.T) { - t.Parallel() - - var h ResponseHeader - - h.Set("set-cookie", "foo=bar; path=/aa/bb; domain=aaa.com") - h.Set(HeaderSetCookie, "aaaaa=bxx") - - var c Cookie - c.SetKey("foo") - if !h.Cookie(&c) { - t.Fatalf("cannot obtain %q cookie", c.Key()) - } - if string(c.Value()) != "bar" { - t.Fatalf("unexpected cookie value %q. Expected %q", c.Value(), "bar") - } - if string(c.Path()) != "/aa/bb" { - t.Fatalf("unexpected cookie path %q. Expected %q", c.Path(), "/aa/bb") - } - if string(c.Domain()) != "aaa.com" { - t.Fatalf("unexpected cookie domain %q. Expected %q", c.Domain(), "aaa.com") - } - - c.SetKey("aaaaa") - if !h.Cookie(&c) { - t.Fatalf("cannot obtain %q cookie", c.Key()) - } - if string(c.Value()) != "bxx" { - t.Fatalf("unexpected cookie value %q. Expecting %q", c.Value(), "bxx") - } -} - -func TestResponseHeaderVisitAll(t *testing.T) { - t.Parallel() - - var h ResponseHeader - - r := bytes.NewBufferString("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Encoding: gzip\r\nContent-Length: 123\r\nSet-Cookie: aa=bb; path=/foo/bar\r\nSet-Cookie: ccc\r\nTrailer: Foo, Bar\r\n\r\n") - br := bufio.NewReader(r) - if err := h.Read(br); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - if h.Len() != 6 { - t.Fatalf("Unexpected number of headers: %d. Expected 6", h.Len()) - } - contentLengthCount := 0 - contentTypeCount := 0 - contentEncodingCount := 0 - cookieCount := 0 - h.VisitAll(func(key, value []byte) { - k := string(key) - v := string(value) - switch k { - case HeaderContentLength: - if v != string(h.Peek(k)) { - t.Fatalf("unexpected content-length: %q. Expecting %q", v, h.Peek(k)) - } - contentLengthCount++ - case HeaderContentType: - if v != string(h.Peek(k)) { - t.Fatalf("Unexpected content-type: %q. Expected %q", v, h.Peek(k)) - } - contentTypeCount++ - case HeaderContentEncoding: - if v != string(h.Peek(k)) { - t.Fatalf("Unexpected content-encoding: %q. Expected %q", v, h.Peek(k)) - } - contentEncodingCount++ - case HeaderSetCookie: - if cookieCount == 0 && v != "aa=bb; path=/foo/bar" { - t.Fatalf("unexpected cookie header: %q. Expected %q", v, "aa=bb; path=/foo/bar") - } - if cookieCount == 1 && v != "ccc" { - t.Fatalf("unexpected cookie header: %q. Expected %q", v, "ccc") - } - cookieCount++ - case HeaderTrailer: - if v != "Foo, Bar" { - t.Fatalf("Unexpected trailer header %q. Expected %q", v, "Foo, Bar") - } - default: - t.Fatalf("unexpected header %q=%q", k, v) - } - }) - if contentLengthCount != 1 { - t.Fatalf("unexpected number of content-length headers: %d. Expected 1", contentLengthCount) - } - if contentTypeCount != 1 { - t.Fatalf("unexpected number of content-type headers: %d. Expected 1", contentTypeCount) - } - if contentEncodingCount != 1 { - t.Fatalf("unexpected number of content-encoding headers: %d. Expected 1", contentEncodingCount) - } - if cookieCount != 2 { - t.Fatalf("unexpected number of cookie header: %d. Expected 2", cookieCount) - } -} - -func TestRequestHeaderVisitAll(t *testing.T) { - t.Parallel() - - var h RequestHeader - - r := bytes.NewBufferString("GET / HTTP/1.1\r\nHost: aa.com\r\nXX: YYY\r\nXX: ZZ\r\nCookie: a=b; c=d\r\nTrailer: Foo, Bar\r\n\r\n") - br := bufio.NewReader(r) - if err := h.Read(br); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - if h.Len() != 5 { - t.Fatalf("Unexpected number of header: %d. Expected 5", h.Len()) - } - hostCount := 0 - xxCount := 0 - cookieCount := 0 - h.VisitAll(func(key, value []byte) { - k := string(key) - v := string(value) - switch k { - case HeaderHost: - if v != string(h.Peek(k)) { - t.Fatalf("Unexpected host value %q. Expected %q", v, h.Peek(k)) - } - hostCount++ - case "Xx": - if xxCount == 0 && v != "YYY" { - t.Fatalf("Unexpected value %q. Expected %q", v, "YYY") - } - if xxCount == 1 && v != "ZZ" { - t.Fatalf("Unexpected value %q. Expected %q", v, "ZZ") - } - xxCount++ - case HeaderCookie: - if v != "a=b; c=d" { - t.Fatalf("Unexpected cookie %q. Expected %q", v, "a=b; c=d") - } - cookieCount++ - case HeaderTrailer: - if v != "Foo, Bar" { - t.Fatalf("Unexpected trailer header %q. Expected %q", v, "Foo, Bar") - } - default: - t.Fatalf("Unexpected header %q=%q", k, v) - } - }) - if hostCount != 1 { - t.Fatalf("Unexpected number of host headers detected %d. Expected 1", hostCount) - } - if xxCount != 2 { - t.Fatalf("Unexpected number of xx headers detected %d. Expected 2", xxCount) - } - if cookieCount != 1 { - t.Fatalf("Unexpected number of cookie headers %d. Expected 1", cookieCount) - } -} - -func TestRequestHeaderVisitAllInOrder(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var h RequestHeader - - r := bytes.NewBufferString("GET / HTTP/1.1\r\nContent-Type: aa\r\nCookie: a=b\r\nHost: example.com\r\nUser-Agent: xxx\r\n\r\n") - br := bufio.NewReader(r) - if err := h.Read(br); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - if h.Len() != 4 { - t.Fatalf("Unexpected number of headers: %d. Expected 4", h.Len()) - } - - order := []string{ - HeaderContentType, - HeaderCookie, - HeaderHost, - HeaderUserAgent, - } - values := []string{ - "aa", - "a=b", - "example.com", - "xxx", - } - - h.VisitAllInOrder(func(key, value []byte) { - if len(order) == 0 { - t.Fatalf("no more headers expected, got %q", key) - } - if order[0] != string(key) { - t.Fatalf("expected header %q got %q", order[0], key) - } - if values[0] != string(value) { - t.Fatalf("expected header value %q got %q", values[0], value) - } - order = order[1:] - values = values[1:] - }) -} - -func TestResponseHeaderAddTrailerError(t *testing.T) { - t.Parallel() - - var h ResponseHeader - err := h.AddTrailer("Foo, Content-Length , Bar,Transfer-Encoding,") - expectedTrailer := "Foo, Bar" - - if !errors.Is(err, ErrBadTrailer) { - t.Fatalf("unexpected err %q. Expected %q", err, ErrBadTrailer) - } - if trailer := string(h.Peek(HeaderTrailer)); trailer != expectedTrailer { - t.Fatalf("unexpected trailer %q. Expected %q", trailer, expectedTrailer) - } - -} - -func TestRequestHeaderAddTrailerError(t *testing.T) { - t.Parallel() - - var h RequestHeader - err := h.AddTrailer("Foo, Content-Length , Bar,Transfer-Encoding,") - expectedTrailer := "Foo, Bar" - - if !errors.Is(err, ErrBadTrailer) { - t.Fatalf("unexpected err %q. Expected %q", err, ErrBadTrailer) - } - if trailer := string(h.Peek(HeaderTrailer)); trailer != expectedTrailer { - t.Fatalf("unexpected trailer %q. Expected %q", trailer, expectedTrailer) - } - -} - -func TestResponseHeaderCookie(t *testing.T) { - t.Parallel() - - var h ResponseHeader - var c Cookie - - c.SetKey("foobar") - c.SetValue("aaa") - h.SetCookie(&c) - - c.SetKey("йцук") - c.SetDomain("foobar.com") - h.SetCookie(&c) - - c.Reset() - c.SetKey("foobar") - if !h.Cookie(&c) { - t.Fatalf("Cannot find cookie %q", c.Key()) - } - - var expectedC1 Cookie - expectedC1.SetKey("foobar") - expectedC1.SetValue("aaa") - if !equalCookie(&expectedC1, &c) { - t.Fatalf("unexpected cookie\n%#v\nExpected\n%#v\n", &c, &expectedC1) - } - - c.SetKey("йцук") - if !h.Cookie(&c) { - t.Fatalf("cannot find cookie %q", c.Key()) - } - - var expectedC2 Cookie - expectedC2.SetKey("йцук") - expectedC2.SetValue("aaa") - expectedC2.SetDomain("foobar.com") - if !equalCookie(&expectedC2, &c) { - t.Fatalf("unexpected cookie\n%v\nExpected\n%v\n", &c, &expectedC2) - } - - h.VisitAllCookie(func(key, value []byte) { - var cc Cookie - if err := cc.ParseBytes(value); err != nil { - t.Fatal(err) - } - if !bytes.Equal(key, cc.Key()) { - t.Fatalf("Unexpected cookie key %q. Expected %q", key, cc.Key()) - } - switch { - case bytes.Equal(key, []byte("foobar")): - if !equalCookie(&expectedC1, &cc) { - t.Fatalf("unexpected cookie\n%v\nExpected\n%v\n", &cc, &expectedC1) - } - case bytes.Equal(key, []byte("йцук")): - if !equalCookie(&expectedC2, &cc) { - t.Fatalf("unexpected cookie\n%v\nExpected\n%v\n", &cc, &expectedC2) - } - default: - t.Fatalf("unexpected cookie key %q", key) - } - }) - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := h.Write(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - h.DelAllCookies() - - var h1 ResponseHeader - br := bufio.NewReader(w) - if err := h1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - c.SetKey("foobar") - if !h1.Cookie(&c) { - t.Fatalf("Cannot find cookie %q", c.Key()) - } - if !equalCookie(&expectedC1, &c) { - t.Fatalf("unexpected cookie\n%v\nExpected\n%v\n", &c, &expectedC1) - } - - h1.DelCookie("foobar") - if h.Cookie(&c) { - t.Fatalf("Unexpected cookie found: %v", &c) - } - if h1.Cookie(&c) { - t.Fatalf("Unexpected cookie found: %v", &c) - } - - c.SetKey("йцук") - if !h1.Cookie(&c) { - t.Fatalf("cannot find cookie %q", c.Key()) - } - if !equalCookie(&expectedC2, &c) { - t.Fatalf("unexpected cookie\n%v\nExpected\n%v\n", &c, &expectedC2) - } - - h1.DelCookie("йцук") - if h.Cookie(&c) { - t.Fatalf("Unexpected cookie found: %v", &c) - } - if h1.Cookie(&c) { - t.Fatalf("Unexpected cookie found: %v", &c) - } -} - -func equalCookie(c1, c2 *Cookie) bool { - if !bytes.Equal(c1.Key(), c2.Key()) { - return false - } - if !bytes.Equal(c1.Value(), c2.Value()) { - return false - } - if !c1.Expire().Equal(c2.Expire()) { - return false - } - if !bytes.Equal(c1.Domain(), c2.Domain()) { - return false - } - if !bytes.Equal(c1.Path(), c2.Path()) { - return false - } - return true -} - -func TestRequestHeaderCookie(t *testing.T) { - t.Parallel() - - var h RequestHeader - h.SetRequestURI("/foobar") - h.Set(HeaderHost, "foobar.com") - - h.SetCookie("foo", "bar") - h.SetCookie("привет", "мир") - - if string(h.Cookie("foo")) != "bar" { - t.Fatalf("Unexpected cookie value %q. Exepcted %q", h.Cookie("foo"), "bar") - } - if string(h.Cookie("привет")) != "мир" { - t.Fatalf("Unexpected cookie value %q. Expected %q", h.Cookie("привет"), "мир") - } - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := h.Write(bw); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - var h1 RequestHeader - br := bufio.NewReader(w) - if err := h1.Read(br); err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - if !bytes.Equal(h1.Cookie("foo"), h.Cookie("foo")) { - t.Fatalf("Unexpected cookie value %q. Exepcted %q", h1.Cookie("foo"), h.Cookie("foo")) - } - h1.DelCookie("foo") - if len(h1.Cookie("foo")) > 0 { - t.Fatalf("Unexpected cookie found: %q", h1.Cookie("foo")) - } - if !bytes.Equal(h1.Cookie("привет"), h.Cookie("привет")) { - t.Fatalf("Unexpected cookie value %q. Expected %q", h1.Cookie("привет"), h.Cookie("привет")) - } - h1.DelCookie("привет") - if len(h1.Cookie("привет")) > 0 { - t.Fatalf("Unexpected cookie found: %q", h1.Cookie("привет")) - } - - h.DelAllCookies() - if len(h.Cookie("foo")) > 0 { - t.Fatalf("Unexpected cookie found: %q", h.Cookie("foo")) - } - if len(h.Cookie("привет")) > 0 { - t.Fatalf("Unexpected cookie found: %q", h.Cookie("привет")) - } -} - -func TestResponseHeaderCookieIssue4(t *testing.T) { - t.Parallel() - - var h ResponseHeader - - c := AcquireCookie() - c.SetKey("foo") - c.SetValue("bar") - h.SetCookie(c) - - if string(h.Peek(HeaderSetCookie)) != "foo=bar" { - t.Fatalf("Unexpected Set-Cookie header %q. Expected %q", h.Peek(HeaderSetCookie), "foo=bar") - } - cookieSeen := false - h.VisitAll(func(key, _ []byte) { - switch string(key) { - case HeaderSetCookie: - cookieSeen = true - } - }) - if !cookieSeen { - t.Fatalf("Set-Cookie not present in VisitAll") - } - - c = AcquireCookie() - c.SetKey("foo") - h.Cookie(c) - if string(c.Value()) != "bar" { - t.Fatalf("Unexpected cookie value %q. Exepcted %q", c.Value(), "bar") - } - - if string(h.Peek(HeaderSetCookie)) != "foo=bar" { - t.Fatalf("Unexpected Set-Cookie header %q. Expected %q", h.Peek(HeaderSetCookie), "foo=bar") - } - cookieSeen = false - h.VisitAll(func(key, _ []byte) { - switch string(key) { - case HeaderSetCookie: - cookieSeen = true - } - }) - if !cookieSeen { - t.Fatalf("Set-Cookie not present in VisitAll") - } -} - -func TestRequestHeaderCookieIssue313(t *testing.T) { - t.Parallel() - - var h RequestHeader - h.SetRequestURI("/") - h.Set(HeaderHost, "foobar.com") - - h.SetCookie("foo", "bar") - - if string(h.Peek(HeaderCookie)) != "foo=bar" { - t.Fatalf("Unexpected Cookie header %q. Expected %q", h.Peek(HeaderCookie), "foo=bar") - } - cookieSeen := false - h.VisitAll(func(key, _ []byte) { - switch string(key) { - case HeaderCookie: - cookieSeen = true - } - }) - if !cookieSeen { - t.Fatalf("Cookie not present in VisitAll") - } - - if string(h.Cookie("foo")) != "bar" { - t.Fatalf("Unexpected cookie value %q. Exepcted %q", h.Cookie("foo"), "bar") - } - - if string(h.Peek(HeaderCookie)) != "foo=bar" { - t.Fatalf("Unexpected Cookie header %q. Expected %q", h.Peek(HeaderCookie), "foo=bar") - } - cookieSeen = false - h.VisitAll(func(key, _ []byte) { - switch string(key) { - case HeaderCookie: - cookieSeen = true - } - }) - if !cookieSeen { - t.Fatalf("Cookie not present in VisitAll") - } -} - -func TestRequestHeaderMethod(t *testing.T) { - t.Parallel() - - // common http methods - testRequestHeaderMethod(t, MethodGet) - testRequestHeaderMethod(t, MethodPost) - testRequestHeaderMethod(t, MethodHead) - testRequestHeaderMethod(t, MethodDelete) - - // non-http methods - testRequestHeaderMethod(t, "foobar") - testRequestHeaderMethod(t, "ABC") -} - -func testRequestHeaderMethod(t *testing.T, expectedMethod string) { - var h RequestHeader - h.SetMethod(expectedMethod) - m := h.Method() - if string(m) != expectedMethod { - t.Fatalf("unexpected method: %q. Expecting %q", m, expectedMethod) - } - - s := h.String() - var h1 RequestHeader - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := h1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - m1 := h1.Method() - if string(m) != string(m1) { - t.Fatalf("unexpected method: %q. Expecting %q", m, m1) - } -} - -func TestRequestHeaderSetGet(t *testing.T) { - t.Parallel() - - h := &RequestHeader{} - h.SetRequestURI("/aa/bbb") - h.SetMethod(MethodPost) - h.Set("foo", "bar") - h.Set("host", "12345") - h.Set("content-type", "aaa/bbb") - h.Set("content-length", "1234") - h.Set("user-agent", "aaabbb") - h.Set("referer", "axcv") - h.Set("baz", "xxxxx") - h.Set("transfer-encoding", "chunked") - h.Set("connection", "close") - - expectRequestHeaderGet(t, h, "Foo", "bar") - expectRequestHeaderGet(t, h, HeaderHost, "12345") - expectRequestHeaderGet(t, h, HeaderContentType, "aaa/bbb") - expectRequestHeaderGet(t, h, HeaderContentLength, "1234") - expectRequestHeaderGet(t, h, "USER-AGent", "aaabbb") - expectRequestHeaderGet(t, h, HeaderReferer, "axcv") - expectRequestHeaderGet(t, h, "baz", "xxxxx") - expectRequestHeaderGet(t, h, HeaderTransferEncoding, "") - expectRequestHeaderGet(t, h, "connecTION", "close") - if !h.ConnectionClose() { - t.Fatalf("unset connection: close") - } - - if h.ContentLength() != 1234 { - t.Fatalf("Unexpected content-length %d. Expected %d", h.ContentLength(), 1234) - } - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - err := h.Write(bw) - if err != nil { - t.Fatalf("Unexpected error when writing request header: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("Unexpected error when flushing request header: %v", err) - } - - var h1 RequestHeader - br := bufio.NewReader(w) - if err = h1.Read(br); err != nil { - t.Fatalf("Unexpected error when reading request header: %v", err) - } - - if h1.ContentLength() != h.ContentLength() { - t.Fatalf("Unexpected Content-Length %d. Expected %d", h1.ContentLength(), h.ContentLength()) - } - - expectRequestHeaderGet(t, &h1, "Foo", "bar") - expectRequestHeaderGet(t, &h1, "HOST", "12345") - expectRequestHeaderGet(t, &h1, HeaderContentType, "aaa/bbb") - expectRequestHeaderGet(t, &h1, HeaderContentLength, "1234") - expectRequestHeaderGet(t, &h1, "USER-AGent", "aaabbb") - expectRequestHeaderGet(t, &h1, HeaderReferer, "axcv") - expectRequestHeaderGet(t, &h1, "baz", "xxxxx") - expectRequestHeaderGet(t, &h1, HeaderTransferEncoding, "") - expectRequestHeaderGet(t, &h1, HeaderConnection, "close") - if !h1.ConnectionClose() { - t.Fatalf("unset connection: close") - } -} - -func TestResponseHeaderSetGet(t *testing.T) { - t.Parallel() - - h := &ResponseHeader{} - h.Set("foo", "bar") - h.Set("content-type", "aaa/bbb") - h.Set("content-encoding", "gzip") - h.Set("connection", "close") - h.Set("content-length", "1234") - h.Set(HeaderServer, "aaaa") - h.Set("baz", "xxxxx") - h.Set(HeaderTransferEncoding, "chunked") - - expectResponseHeaderGet(t, h, "Foo", "bar") - expectResponseHeaderGet(t, h, HeaderContentType, "aaa/bbb") - expectResponseHeaderGet(t, h, HeaderContentEncoding, "gzip") - expectResponseHeaderGet(t, h, HeaderConnection, "close") - expectResponseHeaderGet(t, h, HeaderContentLength, "1234") - expectResponseHeaderGet(t, h, "seRVer", "aaaa") - expectResponseHeaderGet(t, h, "baz", "xxxxx") - expectResponseHeaderGet(t, h, HeaderTransferEncoding, "") - - if h.ContentLength() != 1234 { - t.Fatalf("Unexpected content-length %d. Expected %d", h.ContentLength(), 1234) - } - if !h.ConnectionClose() { - t.Fatalf("Unexpected Connection: close value %v. Expected %v", h.ConnectionClose(), true) - } - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - err := h.Write(bw) - if err != nil { - t.Fatalf("Unexpected error when writing response header: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("Unexpected error when flushing response header: %v", err) - } - - var h1 ResponseHeader - br := bufio.NewReader(w) - if err = h1.Read(br); err != nil { - t.Fatalf("Unexpected error when reading response header: %v", err) - } - - if h1.ContentLength() != h.ContentLength() { - t.Fatalf("Unexpected Content-Length %d. Expected %d", h1.ContentLength(), h.ContentLength()) - } - if h1.ConnectionClose() != h.ConnectionClose() { - t.Fatalf("unexpected connection: close %v. Expected %v", h1.ConnectionClose(), h.ConnectionClose()) - } - - expectResponseHeaderGet(t, &h1, "Foo", "bar") - expectResponseHeaderGet(t, &h1, HeaderContentType, "aaa/bbb") - expectResponseHeaderGet(t, &h1, HeaderContentEncoding, "gzip") - expectResponseHeaderGet(t, &h1, HeaderConnection, "close") - expectResponseHeaderGet(t, &h1, "seRVer", "aaaa") - expectResponseHeaderGet(t, &h1, "baz", "xxxxx") -} - -func expectRequestHeaderGet(t *testing.T, h *RequestHeader, key, expectedValue string) { - if string(h.Peek(key)) != expectedValue { - t.Fatalf("Unexpected value for key %q: %q. Expected %q", key, h.Peek(key), expectedValue) - } -} - -func expectResponseHeaderGet(t *testing.T, h *ResponseHeader, key, expectedValue string) { - if string(h.Peek(key)) != expectedValue { - t.Fatalf("Unexpected value for key %q: %q. Expected %q", key, h.Peek(key), expectedValue) - } -} - -func TestResponseHeaderConnectionClose(t *testing.T) { - t.Parallel() - - testResponseHeaderConnectionClose(t, true) - testResponseHeaderConnectionClose(t, false) -} - -func testResponseHeaderConnectionClose(t *testing.T, connectionClose bool) { - h := &ResponseHeader{} - if connectionClose { - h.SetConnectionClose() - } - h.SetContentLength(123) - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - err := h.Write(bw) - if err != nil { - t.Fatalf("Unexpected error when writing response header: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("Unexpected error when flushing response header: %v", err) - } - - var h1 ResponseHeader - br := bufio.NewReader(w) - err = h1.Read(br) - if err != nil { - t.Fatalf("Unexpected error when reading response header: %v", err) - } - if h1.ConnectionClose() != h.ConnectionClose() { - t.Fatalf("Unexpected value for ConnectionClose: %v. Expected %v", h1.ConnectionClose(), h.ConnectionClose()) - } -} - -func TestRequestHeaderTooBig(t *testing.T) { - t.Parallel() - - s := "GET / HTTP/1.1\r\nHost: aaa.com\r\n" + getHeaders(10500) + "\r\n" - r := bytes.NewBufferString(s) - br := bufio.NewReaderSize(r, 4096) - h := &RequestHeader{} - err := h.Read(br) - if err == nil { - t.Fatalf("Expecting error when reading too big header") - } -} - -func TestResponseHeaderTooBig(t *testing.T) { - t.Parallel() - - s := "HTTP/1.1 200 OK\r\nContent-Type: sss\r\nContent-Length: 0\r\n" + getHeaders(100500) + "\r\n" - r := bytes.NewBufferString(s) - br := bufio.NewReaderSize(r, 4096) - h := &ResponseHeader{} - err := h.Read(br) - if err == nil { - t.Fatalf("Expecting error when reading too big header") - } -} - -type bufioPeekReader struct { - s string - n int -} - -func (r *bufioPeekReader) Read(b []byte) (int, error) { - if len(r.s) == 0 { - return 0, io.EOF - } - - r.n++ - n := r.n - if len(r.s) < n { - n = len(r.s) - } - src := []byte(r.s[:n]) - r.s = r.s[n:] - n = copy(b, src) - return n, nil -} - -func TestRequestHeaderBufioPeek(t *testing.T) { - t.Parallel() - - r := &bufioPeekReader{ - s: "GET / HTTP/1.1\r\nHost: foobar.com\r\n" + getHeaders(10) + "\r\naaaa", - } - br := bufio.NewReaderSize(r, 4096) - h := &RequestHeader{} - if err := h.Read(br); err != nil { - t.Fatalf("Unexpected error when reading request: %v", err) - } - verifyRequestHeader(t, h, -2, "/", "foobar.com", "", "") -} - -func TestResponseHeaderBufioPeek(t *testing.T) { - t.Parallel() - - r := &bufioPeekReader{ - s: "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain\r\nContent-Encoding: gzip\r\n" + getHeaders(10) + "\r\n0123456789", - } - br := bufio.NewReaderSize(r, 4096) - h := &ResponseHeader{} - if err := h.Read(br); err != nil { - t.Fatalf("Unexpected error when reading response: %v", err) - } - verifyResponseHeader(t, h, 200, 10, "text/plain", "gzip") -} - -func getHeaders(n int) string { - var h []string - for i := 0; i < n; i++ { - h = append(h, fmt.Sprintf("Header_%d: Value_%d\r\n", i, i)) - } - return strings.Join(h, "") -} - -func TestResponseHeaderReadSuccess(t *testing.T) { - t.Parallel() - - h := &ResponseHeader{} - - // straight order of content-length and content-type - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n\r\n", - 200, 123, "text/html") - if h.ConnectionClose() { - t.Fatalf("unexpected connection: close") - } - - // reverse order of content-length and content-type - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 202 OK\r\nContent-Type: text/plain; encoding=utf-8\r\nContent-Length: 543\r\nConnection: close\r\n\r\n", - 202, 543, "text/plain; encoding=utf-8") - if !h.ConnectionClose() { - t.Fatalf("expecting connection: close") - } - - // tranfer-encoding: chunked - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 505 Internal error\r\nContent-Type: text/html\r\nTransfer-Encoding: chunked\r\n\r\n", - 505, -1, "text/html") - if h.ConnectionClose() { - t.Fatalf("unexpected connection: close") - } - - // reverse order of content-type and tranfer-encoding - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 343 foobar\r\nTransfer-Encoding: chunked\r\nContent-Type: text/json\r\n\r\n", - 343, -1, "text/json") - - // additional headers - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 100 Continue\r\nFoobar: baz\r\nContent-Type: aaa/bbb\r\nUser-Agent: x\r\nContent-Length: 123\r\nZZZ: werer\r\n\r\n", - 100, 123, "aaa/bbb") - - // ancient http protocol - testResponseHeaderReadSuccess(t, h, "HTTP/0.9 300 OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n\r\nqqqq", - 300, 123, "text/html") - - // lf instead of crlf - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\nContent-Length: 123\nContent-Type: text/html\n\n", - 200, 123, "text/html") - - // Zero-length headers with mixed crlf and lf - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 400 OK\nContent-Length: 345\nZero-Value: \r\nContent-Type: aaa\n: zero-key\r\n\r\nooa", - 400, 345, "aaa") - - // No space after colon - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\nContent-Length:34\nContent-Type: sss\n\naaaa", - 200, 34, "sss") - - // invalid case - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 400 OK\nconTEnt-leNGTH: 123\nConTENT-TYPE: ass\n\n", - 400, 123, "ass") - - // duplicate content-length - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\nContent-Length: 456\r\nContent-Type: foo/bar\r\nContent-Length: 321\r\n\r\n", - 200, 321, "foo/bar") - - // duplicate content-type - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\nContent-Length: 234\r\nContent-Type: foo/bar\r\nContent-Type: baz/bar\r\n\r\n", - 200, 234, "baz/bar") - - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 300 OK\r\nContent-Type: foo/barr\r\nTransfer-Encoding: chunked\r\nContent-Length: 354\r\n\r\n", - 300, -1, "foo/barr") - - // duplicate transfer-encoding: chunked - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nTransfer-Encoding: chunked\r\nTransfer-Encoding: chunked\r\n\r\n", - 200, -1, "text/html") - - // no reason string in the first line - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 456\r\nContent-Type: xxx/yyy\r\nContent-Length: 134\r\n\r\naaaxxx", - 456, 134, "xxx/yyy") - - // blank lines before the first line - testResponseHeaderReadSuccess(t, h, "\r\nHTTP/1.1 200 OK\r\nContent-Type: aa\r\nContent-Length: 0\r\n\r\nsss", - 200, 0, "aa") - if h.ConnectionClose() { - t.Fatalf("unexpected connection: close") - } - - // no content-length (informational responses) - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 101 OK\r\n\r\n", - 101, -2, "text/plain; charset=utf-8") - if h.ConnectionClose() { - t.Fatalf("expecting connection: keep-alive for informational response") - } - - // no content-length (no-content responses) - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 204 OK\r\n\r\n", - 204, -2, "text/plain; charset=utf-8") - if h.ConnectionClose() { - t.Fatalf("expecting connection: keep-alive for no-content response") - } - - // no content-length (not-modified responses) - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 304 OK\r\n\r\n", - 304, -2, "text/plain; charset=utf-8") - if h.ConnectionClose() { - t.Fatalf("expecting connection: keep-alive for not-modified response") - } - - // no content-length (identity transfer-encoding) - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\nContent-Type: foo/bar\r\n\r\nabcdefg", - 200, -2, "foo/bar") - if !h.ConnectionClose() { - t.Fatalf("expecting connection: close for identity response") - } - - // no content-type - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 400 OK\r\nContent-Length: 123\r\n\r\nfoiaaa", - 400, 123, string(defaultContentType)) - - // no content-type and no default - h.SetNoDefaultContentType(true) - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 400 OK\r\nContent-Length: 123\r\n\r\nfoiaaa", - 400, 123, "") - h.SetNoDefaultContentType(false) - - // no headers - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\n\r\naaaabbb", - 200, -2, string(defaultContentType)) - if !h.IsHTTP11() { - t.Fatalf("expecting http/1.1 protocol") - } - - // ancient http protocol - testResponseHeaderReadSuccess(t, h, "HTTP/1.0 203 OK\r\nContent-Length: 123\r\nContent-Type: foobar\r\n\r\naaa", - 203, 123, "foobar") - if h.IsHTTP11() { - t.Fatalf("ancient protocol must be non-http/1.1") - } - if !h.ConnectionClose() { - t.Fatalf("expecting connection: close for ancient protocol") - } - - // ancient http protocol with 'Connection: keep-alive' header. - testResponseHeaderReadSuccess(t, h, "HTTP/1.0 403 aa\r\nContent-Length: 0\r\nContent-Type: 2\r\nConnection: Keep-Alive\r\n\r\nww", - 403, 0, "2") - if h.IsHTTP11() { - t.Fatalf("ancient protocol must be non-http/1.1") - } - if h.ConnectionClose() { - t.Fatalf("expecting connection: keep-alive for ancient protocol") - } -} - -func TestRequestHeaderReadSuccess(t *testing.T) { - t.Parallel() - - h := &RequestHeader{} - - // simple headers - testRequestHeaderReadSuccess(t, h, "GET /foo/bar HTTP/1.1\r\nHost: google.com\r\n\r\n", - -2, "/foo/bar", "google.com", "", "", nil) - if h.ConnectionClose() { - t.Fatalf("unexpected connection: close header") - } - - // simple headers with body - testRequestHeaderReadSuccess(t, h, "GET /a/bar HTTP/1.1\r\nHost: gole.com\r\nconneCTION: close\r\n\r\nfoobar", - -2, "/a/bar", "gole.com", "", "", nil) - if !h.ConnectionClose() { - t.Fatalf("connection: close unset") - } - - // ancient http protocol - testRequestHeaderReadSuccess(t, h, "GET /bar HTTP/1.0\r\nHost: gole\r\n\r\npppp", - -2, "/bar", "gole", "", "", nil) - if h.IsHTTP11() { - t.Fatalf("ancient http protocol cannot be http/1.1") - } - if !h.ConnectionClose() { - t.Fatalf("expecting connectionClose for ancient http protocol") - } - - // ancient http protocol with 'Connection: keep-alive' header - testRequestHeaderReadSuccess(t, h, "GET /aa HTTP/1.0\r\nHost: bb\r\nConnection: keep-alive\r\n\r\nxxx", - -2, "/aa", "bb", "", "", nil) - if h.IsHTTP11() { - t.Fatalf("ancient http protocol cannot be http/1.1") - } - if h.ConnectionClose() { - t.Fatalf("unexpected 'connection: close' for ancient http protocol") - } - - // complex headers with body - testRequestHeaderReadSuccess(t, h, "GET /aabar HTTP/1.1\r\nAAA: bbb\r\nHost: ole.com\r\nAA: bb\r\n\r\nzzz", - -2, "/aabar", "ole.com", "", "", nil) - if !h.IsHTTP11() { - t.Fatalf("expecting http/1.1 protocol") - } - if h.ConnectionClose() { - t.Fatalf("unexpected connection: close") - } - - // lf instead of crlf - testRequestHeaderReadSuccess(t, h, "GET /foo/bar HTTP/1.1\nHost: google.com\n\n", - -2, "/foo/bar", "google.com", "", "", nil) - - // post method - testRequestHeaderReadSuccess(t, h, "POST /aaa?bbb HTTP/1.1\r\nHost: foobar.com\r\nContent-Length: 1235\r\nContent-Type: aaa\r\n\r\nabcdef", - 1235, "/aaa?bbb", "foobar.com", "", "aaa", nil) - - // zero-length headers with mixed crlf and lf - testRequestHeaderReadSuccess(t, h, "GET /a HTTP/1.1\nHost: aaa\r\nZero: \n: Zero-Value\n\r\nxccv", - -2, "/a", "aaa", "", "", nil) - - // no space after colon - testRequestHeaderReadSuccess(t, h, "GET /a HTTP/1.1\nHost:aaaxd\n\nsdfds", - -2, "/a", "aaaxd", "", "", nil) - - // get with zero content-length - testRequestHeaderReadSuccess(t, h, "GET /xxx HTTP/1.1\nHost: aaa.com\nContent-Length: 0\n\n", - 0, "/xxx", "aaa.com", "", "", nil) - - // get with non-zero content-length - testRequestHeaderReadSuccess(t, h, "GET /xxx HTTP/1.1\nHost: aaa.com\nContent-Length: 123\n\n", - 123, "/xxx", "aaa.com", "", "", nil) - - // invalid case - testRequestHeaderReadSuccess(t, h, "GET /aaa HTTP/1.1\nhoST: bbb.com\n\naas", - -2, "/aaa", "bbb.com", "", "", nil) - - // referer - testRequestHeaderReadSuccess(t, h, "GET /asdf HTTP/1.1\nHost: aaa.com\nReferer: bb.com\n\naaa", - -2, "/asdf", "aaa.com", "bb.com", "", nil) - - // duplicate host - testRequestHeaderReadSuccess(t, h, "GET /aa HTTP/1.1\r\nHost: aaaaaa.com\r\nHost: bb.com\r\n\r\n", - -2, "/aa", "bb.com", "", "", nil) - - // post with duplicate content-type - testRequestHeaderReadSuccess(t, h, "POST /a HTTP/1.1\r\nHost: aa\r\nContent-Type: ab\r\nContent-Length: 123\r\nContent-Type: xx\r\n\r\n", - 123, "/a", "aa", "", "xx", nil) - - // post with duplicate content-length - testRequestHeaderReadSuccess(t, h, "POST /xx HTTP/1.1\r\nHost: aa\r\nContent-Type: s\r\nContent-Length: 13\r\nContent-Length: 1\r\n\r\n", - 1, "/xx", "aa", "", "s", nil) - - // non-post with content-type - testRequestHeaderReadSuccess(t, h, "GET /aaa HTTP/1.1\r\nHost: bbb.com\r\nContent-Type: aaab\r\n\r\n", - -2, "/aaa", "bbb.com", "", "aaab", nil) - - // non-post with content-length - testRequestHeaderReadSuccess(t, h, "HEAD / HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: 123\r\n\r\n", - 123, "/", "aaa.com", "", "", nil) - - // non-post with content-type and content-length - testRequestHeaderReadSuccess(t, h, "GET /aa HTTP/1.1\r\nHost: aa.com\r\nContent-Type: abd/test\r\nContent-Length: 123\r\n\r\n", - 123, "/aa", "aa.com", "", "abd/test", nil) - - // request uri with hostname - testRequestHeaderReadSuccess(t, h, "GET http://gooGle.com/foO/%20bar?xxx#aaa HTTP/1.1\r\nHost: aa.cOM\r\n\r\ntrail", - -2, "http://gooGle.com/foO/%20bar?xxx#aaa", "aa.cOM", "", "", nil) - - // no protocol in the first line - testRequestHeaderReadSuccess(t, h, "GET /foo/bar\r\nHost: google.com\r\n\r\nisdD", - -2, "/foo/bar", "google.com", "", "", nil) - - // blank lines before the first line - testRequestHeaderReadSuccess(t, h, "\r\n\n\r\nGET /aaa HTTP/1.1\r\nHost: aaa.com\r\n\r\nsss", - -2, "/aaa", "aaa.com", "", "", nil) - - // request uri with spaces - testRequestHeaderReadSuccess(t, h, "GET /foo/ bar baz HTTP/1.1\r\nHost: aa.com\r\n\r\nxxx", - -2, "/foo/ bar baz", "aa.com", "", "", nil) - - // no host - testRequestHeaderReadSuccess(t, h, "GET /foo/bar HTTP/1.1\r\nFOObar: assdfd\r\n\r\naaa", - -2, "/foo/bar", "", "", "", nil) - - // no host, no headers - testRequestHeaderReadSuccess(t, h, "GET /foo/bar HTTP/1.1\r\n\r\nfoobar", - -2, "/foo/bar", "", "", "", nil) - - // post without content-length and content-type - testRequestHeaderReadSuccess(t, h, "POST /aaa HTTP/1.1\r\nHost: aaa.com\r\n\r\nzxc", - -2, "/aaa", "aaa.com", "", "", nil) - - // post without content-type - testRequestHeaderReadSuccess(t, h, "POST /abc HTTP/1.1\r\nHost: aa.com\r\nContent-Length: 123\r\n\r\npoiuy", - 123, "/abc", "aa.com", "", "", nil) - - // post without content-length - testRequestHeaderReadSuccess(t, h, "POST /abc HTTP/1.1\r\nHost: aa.com\r\nContent-Type: adv\r\n\r\n123456", - -2, "/abc", "aa.com", "", "adv", nil) - - // invalid method - testRequestHeaderReadSuccess(t, h, "POST /foo/bar HTTP/1.1\r\nHost: google.com\r\n\r\nmnbv", - -2, "/foo/bar", "google.com", "", "", nil) - - // put request - testRequestHeaderReadSuccess(t, h, "PUT /faa HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: 123\r\nContent-Type: aaa\r\n\r\nxwwere", - 123, "/faa", "aaa.com", "", "aaa", nil) -} - -func TestResponseHeaderReadError(t *testing.T) { - t.Parallel() - - h := &ResponseHeader{} - - // incorrect first line - testResponseHeaderReadError(t, h, "") - testResponseHeaderReadError(t, h, "fo") - testResponseHeaderReadError(t, h, "foobarbaz") - testResponseHeaderReadError(t, h, "HTTP/1.1") - testResponseHeaderReadError(t, h, "HTTP/1.1 ") - testResponseHeaderReadError(t, h, "HTTP/1.1 s") - - // non-numeric status code - testResponseHeaderReadError(t, h, "HTTP/1.1 foobar OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n\r\n") - testResponseHeaderReadError(t, h, "HTTP/1.1 123foobar OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n\r\n") - testResponseHeaderReadError(t, h, "HTTP/1.1 foobar344 OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n\r\n") - - // non-numeric content-length - testResponseHeaderReadError(t, h, "HTTP/1.1 200 OK\r\nContent-Length: faaa\r\nContent-Type: text/html\r\n\r\nfoobar") - testResponseHeaderReadError(t, h, "HTTP/1.1 201 OK\r\nContent-Length: 123aa\r\nContent-Type: text/ht\r\n\r\naaa") - testResponseHeaderReadError(t, h, "HTTP/1.1 200 OK\r\nContent-Length: aa124\r\nContent-Type: html\r\n\r\nxx") - - // no headers - testResponseHeaderReadError(t, h, "HTTP/1.1 200 OK\r\n") - - // no trailing crlf - testResponseHeaderReadError(t, h, "HTTP/1.1 200 OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n") - - // forbidden trailer - testResponseHeaderReadError(t, h, "HTTP/1.1 200 OK\r\nContent-Length: -1\r\nTrailer: Foo, Content-Length\r\n\r\n") -} - -func TestResponseHeaderReadErrorSecureLog(t *testing.T) { - t.Parallel() - - h := &ResponseHeader{ - secureErrorLogMessage: true, - } - - // incorrect first line - testResponseHeaderReadSecuredError(t, h, "fo") - testResponseHeaderReadSecuredError(t, h, "foobarbaz") - testResponseHeaderReadSecuredError(t, h, "HTTP/1.1") - testResponseHeaderReadSecuredError(t, h, "HTTP/1.1 ") - testResponseHeaderReadSecuredError(t, h, "HTTP/1.1 s") - - // non-numeric status code - testResponseHeaderReadSecuredError(t, h, "HTTP/1.1 foobar OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n\r\n") - testResponseHeaderReadSecuredError(t, h, "HTTP/1.1 123foobar OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n\r\n") - testResponseHeaderReadSecuredError(t, h, "HTTP/1.1 foobar344 OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n\r\n") - - // no headers - testResponseHeaderReadSecuredError(t, h, "HTTP/1.1 200 OK\r\n") - - // no trailing crlf - testResponseHeaderReadSecuredError(t, h, "HTTP/1.1 200 OK\r\nContent-Length: 123\r\nContent-Type: text/html\r\n") -} - -func TestRequestHeaderReadError(t *testing.T) { - t.Parallel() - - h := &RequestHeader{} - - // incorrect first line - testRequestHeaderReadError(t, h, "") - testRequestHeaderReadError(t, h, "fo") - testRequestHeaderReadError(t, h, "GET ") - testRequestHeaderReadError(t, h, "GET / HTTP/1.1\r") - - // missing RequestURI - testRequestHeaderReadError(t, h, "GET HTTP/1.1\r\nHost: google.com\r\n\r\n") - - // post with invalid content-length - testRequestHeaderReadError(t, h, "POST /a HTTP/1.1\r\nHost: bb\r\nContent-Type: aa\r\nContent-Length: dff\r\n\r\nqwerty") - - // forbidden trailer - testRequestHeaderReadError(t, h, "POST /a HTTP/1.1\r\nContent-Length: -1\r\nTrailer: Foo, Content-Length\r\n\r\n") -} - -func TestRequestHeaderReadSecuredError(t *testing.T) { - t.Parallel() - - h := &RequestHeader{ - secureErrorLogMessage: true, - } - - // incorrect first line - testRequestHeaderReadSecuredError(t, h, "fo") - testRequestHeaderReadSecuredError(t, h, "GET ") - testRequestHeaderReadSecuredError(t, h, "GET / HTTP/1.1\r") - - // missing RequestURI - testRequestHeaderReadSecuredError(t, h, "GET HTTP/1.1\r\nHost: google.com\r\n\r\n") - - // post with invalid content-length - testRequestHeaderReadSecuredError(t, h, "POST /a HTTP/1.1\r\nHost: bb\r\nContent-Type: aa\r\nContent-Length: dff\r\n\r\nqwerty") -} - -func testResponseHeaderReadError(t *testing.T, h *ResponseHeader, headers string) { - r := bytes.NewBufferString(headers) - br := bufio.NewReader(r) - err := h.Read(br) - if err == nil { - t.Fatalf("Expecting error when reading response header %q", headers) - } - // make sure response header works after error - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\nContent-Type: foo/bar\r\nContent-Length: 12345\r\n\r\nsss", - 200, 12345, "foo/bar") -} - -func testResponseHeaderReadSecuredError(t *testing.T, h *ResponseHeader, headers string) { - r := bytes.NewBufferString(headers) - br := bufio.NewReader(r) - err := h.Read(br) - if err == nil { - t.Fatalf("Expecting error when reading response header %q", headers) - } - if strings.Contains(err.Error(), headers) { - t.Fatalf("Not expecting header content in err %q", err) - } - // make sure response header works after error - testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\nContent-Type: foo/bar\r\nContent-Length: 12345\r\n\r\nsss", - 200, 12345, "foo/bar") -} - -func testRequestHeaderReadError(t *testing.T, h *RequestHeader, headers string) { - r := bytes.NewBufferString(headers) - br := bufio.NewReader(r) - err := h.Read(br) - if err == nil { - t.Fatalf("Expecting error when reading request header %q", headers) - } - - // make sure request header works after error - testRequestHeaderReadSuccess(t, h, "GET /foo/bar HTTP/1.1\r\nHost: aaaa\r\n\r\nxxx", - -2, "/foo/bar", "aaaa", "", "", nil) -} - -func testRequestHeaderReadSecuredError(t *testing.T, h *RequestHeader, headers string) { - r := bytes.NewBufferString(headers) - br := bufio.NewReader(r) - err := h.Read(br) - if err == nil { - t.Fatalf("Expecting error when reading request header %q", headers) - } - if strings.Contains(err.Error(), headers) { - t.Fatalf("Not expecting header content in err %q", err) - } - // make sure request header works after error - testRequestHeaderReadSuccess(t, h, "GET /foo/bar HTTP/1.1\r\nHost: aaaa\r\n\r\nxxx", - -2, "/foo/bar", "aaaa", "", "", nil) -} - -func testResponseHeaderReadSuccess(t *testing.T, h *ResponseHeader, headers string, expectedStatusCode, expectedContentLength int, - expectedContentType string) { - r := bytes.NewBufferString(headers) - br := bufio.NewReader(r) - err := h.Read(br) - if err != nil { - t.Fatalf("Unexpected error when parsing response headers: %v. headers=%q", err, headers) - } - verifyResponseHeader(t, h, expectedStatusCode, expectedContentLength, expectedContentType, "") -} - -func testRequestHeaderReadSuccess(t *testing.T, h *RequestHeader, headers string, expectedContentLength int, - expectedRequestURI, expectedHost, expectedReferer, expectedContentType string, expectedTrailer map[string]string) { - r := bytes.NewBufferString(headers) - br := bufio.NewReader(r) - err := h.Read(br) - if err != nil { - t.Fatalf("Unexpected error when parsing request headers: %v. headers=%q", err, headers) - } - verifyRequestHeader(t, h, expectedContentLength, expectedRequestURI, expectedHost, expectedReferer, expectedContentType) -} - -func verifyResponseHeader(t *testing.T, h *ResponseHeader, expectedStatusCode, expectedContentLength int, expectedContentType, expectedContentEncoding string) { - if h.StatusCode() != expectedStatusCode { - t.Fatalf("Unexpected status code %d. Expected %d", h.StatusCode(), expectedStatusCode) - } - if h.ContentLength() != expectedContentLength { - t.Fatalf("Unexpected content length %d. Expected %d", h.ContentLength(), expectedContentLength) - } - if string(h.ContentType()) != expectedContentType { - t.Fatalf("Unexpected content type %q. Expected %q", h.ContentType(), expectedContentType) - } - if string(h.ContentEncoding()) != expectedContentEncoding { - t.Fatalf("Unexpected content encoding %q. Expected %q", h.ContentEncoding(), expectedContentEncoding) - } -} - -func verifyResponseHeaderConnection(t *testing.T, h *ResponseHeader, expectConnection string) { - if string(h.Peek(HeaderConnection)) != expectConnection { - t.Fatalf("Unexpected Connection %q. Expected %q", h.Peek(HeaderConnection), expectConnection) - } -} - -func verifyRequestHeader(t *testing.T, h *RequestHeader, expectedContentLength int, - expectedRequestURI, expectedHost, expectedReferer, expectedContentType string) { - if h.ContentLength() != expectedContentLength { - t.Fatalf("Unexpected Content-Length %d. Expected %d", h.ContentLength(), expectedContentLength) - } - if string(h.RequestURI()) != expectedRequestURI { - t.Fatalf("Unexpected RequestURI %q. Expected %q", h.RequestURI(), expectedRequestURI) - } - if string(h.Peek(HeaderHost)) != expectedHost { - t.Fatalf("Unexpected host %q. Expected %q", h.Peek(HeaderHost), expectedHost) - } - if string(h.Peek(HeaderReferer)) != expectedReferer { - t.Fatalf("Unexpected referer %q. Expected %q", h.Peek(HeaderReferer), expectedReferer) - } - if string(h.Peek(HeaderContentType)) != expectedContentType { - t.Fatalf("Unexpected content-type %q. Expected %q", h.Peek(HeaderContentType), expectedContentType) - } -} - -func verifyResponseTrailer(t *testing.T, h *ResponseHeader, expectedTrailers map[string]string) { - for k, v := range expectedTrailers { - got := h.Peek(k) - if !bytes.Equal(got, []byte(v)) { - t.Fatalf("Unexpected trailer %q. Expected %q. Got %q", k, v, got) - } - } -} - -func verifyRequestTrailer(t *testing.T, h *RequestHeader, expectedTrailers map[string]string) { - for k, v := range expectedTrailers { - got := h.Peek(k) - if !bytes.Equal(got, []byte(v)) { - t.Fatalf("Unexpected trailer %q. Expected %q. Got %q", k, v, got) - } - } -} - -func verifyTrailer(t *testing.T, r *bufio.Reader, expectedTrailers map[string]string, isReq bool) { - if isReq { - req := Request{} - err := req.Header.ReadTrailer(r) - if err == io.EOF && expectedTrailers == nil { - return - } - if err != nil { - t.Fatalf("Cannot read trailer: %v", err) - } - verifyRequestTrailer(t, &req.Header, expectedTrailers) - return - } - - resp := Response{} - err := resp.Header.ReadTrailer(r) - if err == io.EOF && expectedTrailers == nil { - return - } - if err != nil { - t.Fatalf("Cannot read trailer: %v", err) - } - verifyResponseTrailer(t, &resp.Header, expectedTrailers) -} diff --git a/lib/fasthttp/header_timing_test.go b/lib/fasthttp/header_timing_test.go deleted file mode 100644 index 3e78a9992..000000000 --- a/lib/fasthttp/header_timing_test.go +++ /dev/null @@ -1,221 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "io" - "strconv" - "testing" - - "infini.sh/framework/lib/bytebufferpool" -) - -var strFoobar = []byte("foobar.com") - -// it has the same length as Content-Type -var strNonSpecialHeader = []byte("Dontent-Type") - -type benchReadBuf struct { - s []byte - n int -} - -func (r *benchReadBuf) Read(p []byte) (int, error) { - if r.n == len(r.s) { - return 0, io.EOF - } - - n := copy(p, r.s[r.n:]) - r.n += n - return n, nil -} - -func BenchmarkRequestHeaderRead(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var h RequestHeader - buf := &benchReadBuf{ - s: []byte("GET /foo/bar HTTP/1.1\r\nHost: foobar.com\r\nUser-Agent: aaa.bbb\r\nReferer: http://google.com/aaa/bbb\r\n\r\n"), - } - br := bufio.NewReader(buf) - for pb.Next() { - buf.n = 0 - br.Reset(buf) - if err := h.Read(br); err != nil { - b.Fatalf("unexpected error when reading header: %v", err) - } - } - }) -} - -func BenchmarkResponseHeaderRead(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var h ResponseHeader - buf := &benchReadBuf{ - s: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 1256\r\nServer: aaa 1/2.3\r\nTest: 1.2.3\r\n\r\n"), - } - br := bufio.NewReader(buf) - for pb.Next() { - buf.n = 0 - br.Reset(buf) - if err := h.Read(br); err != nil { - b.Fatalf("unexpected error when reading header: %v", err) - } - } - }) -} - -func BenchmarkRequestHeaderWrite(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var h RequestHeader - h.SetRequestURI("/foo/bar") - h.SetHost("foobar.com") - h.SetUserAgent("aaa.bbb") - h.SetReferer("http://google.com/aaa/bbb") - var w bytebufferpool.ByteBuffer - for pb.Next() { - if _, err := h.WriteTo(&w); err != nil { - b.Fatalf("unexpected error when writing header: %v", err) - } - w.Reset() - } - }) -} - -func BenchmarkResponseHeaderWrite(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var h ResponseHeader - h.SetStatusCode(200) - h.SetContentType("text/html") - h.SetContentLength(1256) - h.SetServer("aaa 1/2.3") - h.Set("Test", "1.2.3") - var w bytebufferpool.ByteBuffer - for pb.Next() { - if _, err := h.WriteTo(&w); err != nil { - b.Fatalf("unexpected error when writing header: %v", err) - } - w.Reset() - } - }) -} - -// Result: 2.2 ns/op -func BenchmarkRequestHeaderPeekBytesSpecialHeader(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var h RequestHeader - h.SetContentTypeBytes(strFoobar) - for pb.Next() { - v := h.PeekBytes(strContentType) - if !bytes.Equal(v, strFoobar) { - b.Fatalf("unexpected result: %q. Expected %q", v, strFoobar) - } - } - }) -} - -// Result: 2.9 ns/op -func BenchmarkRequestHeaderPeekBytesNonSpecialHeader(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var h RequestHeader - h.SetBytesKV(strNonSpecialHeader, strFoobar) - for pb.Next() { - v := h.PeekBytes(strNonSpecialHeader) - if !bytes.Equal(v, strFoobar) { - b.Fatalf("unexpected result: %q. Expected %q", v, strFoobar) - } - } - }) -} - -// Result: 2.3 ns/op -func BenchmarkResponseHeaderPeekBytesSpecialHeader(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var h ResponseHeader - h.SetContentTypeBytes(strFoobar) - for pb.Next() { - v := h.PeekBytes(strContentType) - if !bytes.Equal(v, strFoobar) { - b.Fatalf("unexpected result: %q. Expected %q", v, strFoobar) - } - } - }) -} - -// Result: 2.9 ns/op -func BenchmarkResponseHeaderPeekBytesNonSpecialHeader(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var h ResponseHeader - h.SetBytesKV(strNonSpecialHeader, strFoobar) - for pb.Next() { - v := h.PeekBytes(strNonSpecialHeader) - if !bytes.Equal(v, strFoobar) { - b.Fatalf("unexpected result: %q. Expected %q", v, strFoobar) - } - } - }) -} - -func BenchmarkNormalizeHeaderKeyCommonCase(b *testing.B) { - src := []byte("User-Agent-Host-Content-Type-Content-Length-Server") - benchmarkNormalizeHeaderKey(b, src) -} - -func BenchmarkNormalizeHeaderKeyLowercase(b *testing.B) { - src := []byte("user-agent-host-content-type-content-length-server") - benchmarkNormalizeHeaderKey(b, src) -} - -func BenchmarkNormalizeHeaderKeyUppercase(b *testing.B) { - src := []byte("USER-AGENT-HOST-CONTENT-TYPE-CONTENT-LENGTH-SERVER") - benchmarkNormalizeHeaderKey(b, src) -} - -func benchmarkNormalizeHeaderKey(b *testing.B, src []byte) { - b.RunParallel(func(pb *testing.PB) { - buf := make([]byte, len(src)) - for pb.Next() { - copy(buf, src) - normalizeHeaderKey(buf, false) - } - }) -} - -func BenchmarkRemoveNewLines(b *testing.B) { - type testcase struct { - value string - expectedValue string - } - - var testcases = []testcase{ - {value: "MaliciousValue", expectedValue: "MaliciousValue"}, - {value: "MaliciousValue\r\n", expectedValue: "MaliciousValue "}, - {value: "Malicious\nValue", expectedValue: "Malicious Value"}, - {value: "Malicious\rValue", expectedValue: "Malicious Value"}, - } - - for i, tcase := range testcases { - caseName := strconv.FormatInt(int64(i), 10) - b.Run(caseName, func(subB *testing.B) { - subB.ReportAllocs() - var h RequestHeader - for i := 0; i < subB.N; i++ { - h.Set("Test", tcase.value) - } - subB.StopTimer() - actualValue := string(h.Peek("Test")) - - if actualValue != tcase.expectedValue { - subB.Errorf("unexpected value, got: %+v", actualValue) - } - }) - } -} - -func BenchmarkRequestHeaderIsGet(b *testing.B) { - req := &RequestHeader{method: []byte(MethodGet)} - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - req.IsGet() - } - }) -} diff --git a/lib/fasthttp/http_test.go b/lib/fasthttp/http_test.go deleted file mode 100644 index cbcadfec0..000000000 --- a/lib/fasthttp/http_test.go +++ /dev/null @@ -1,2966 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "encoding/base64" - "errors" - "fmt" - "io" - "io/ioutil" - "math" - "mime/multipart" - "net/http" - "net/http/httptest" - "os" - "reflect" - "strconv" - "strings" - "testing" - "time" - - "infini.sh/framework/lib/bytebufferpool" -) - -func TestInvalidTrailers(t *testing.T) { - t.Parallel() - - if err := (&Response{}).Read(bufio.NewReader(bytes.NewReader([]byte{0x20, 0x30, 0x0a, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2d, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x3a, 0xff, 0x0a, 0x0a, 0x30, 0x0d, 0x0a, 0x30}))); !errors.Is(err, io.EOF) { - t.Fatalf("%#v", err) - } - if err := (&Response{}).Read(bufio.NewReader(bytes.NewReader([]byte{0xff, 0x20, 0x0a, 0x54, 0x52, 0x61, 0x49, 0x4c, 0x65, 0x52, 0x3a, 0x2c, 0x0a, 0x0a}))); !errors.Is(err, errEmptyInt) { - t.Fatal(err) - } - if err := (&Response{}).Read(bufio.NewReader(bytes.NewReader([]byte{0x54, 0x52, 0x61, 0x49, 0x4c, 0x65, 0x52, 0x3a, 0x2c, 0x0a, 0x0a}))); !strings.Contains(err.Error(), "cannot find whitespace in the first line of response") { - t.Fatal(err) - } - if err := (&Request{}).Read(bufio.NewReader(bytes.NewReader([]byte{0xff, 0x20, 0x0a, 0x54, 0x52, 0x61, 0x49, 0x4c, 0x65, 0x52, 0x3a, 0x2c, 0x0a, 0x0a}))); !strings.Contains(err.Error(), "contain forbidden trailer") { - t.Fatal(err) - } - - b, _ := base64.StdEncoding.DecodeString("tCAKIDoKCToKICAKCToKICAKCToKIAogOgoJOgogIAoJOgovIC8vOi4KOh0KVFJhSUxlUjo9HT09HQpUUmFJTGVSOicQAApUUmFJTGVSOj0gHSAKCT09HQoKOgoKCgo=") - if err := (&Request{}).Read(bufio.NewReader(bytes.NewReader(b))); !strings.Contains(err.Error(), "error when reading request headers: invalid header key") { - t.Fatalf("%#v", err) - } -} - -func TestResponseEmptyTransferEncoding(t *testing.T) { - t.Parallel() - - var r Response - - body := "Some body" - br := bufio.NewReader(bytes.NewBufferString("HTTP/1.1 200 OK\r\nContent-Type: aaa\r\nTransfer-Encoding: \r\nContent-Length: 9\r\n\r\n" + body)) - err := r.Read(br) - if err != nil { - t.Fatal(err) - } - if got := string(r.Body()); got != body { - t.Fatalf("expected %q got %q", body, got) - } -} - -// Don't send the fragment/hash/# part of a URL to the server. -func TestFragmentInURIRequest(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var req Request - req.SetRequestURI("https://docs.gitlab.com/ee/user/project/integrations/webhooks.html#events") - - var b bytes.Buffer - req.WriteTo(&b) //nolint:errcheck - got := b.String() - expected := "GET /ee/user/project/integrations/webhooks.html HTTP/1.1\r\nHost: docs.gitlab.com\r\n\r\n" - - if got != expected { - t.Errorf("got %q expected %q", got, expected) - } -} - -func TestIssue875(t *testing.T) { - t.Parallel() - - type testcase struct { - uri string - expectedRedirect string - expectedLocation string - } - - testcases := []testcase{ - { - uri: `http://localhost:3000/?redirect=foo%0d%0aSet-Cookie:%20SESSIONID=MaliciousValue%0d%0a`, - expectedRedirect: "foo\r\nSet-Cookie: SESSIONID=MaliciousValue\r\n", - expectedLocation: "Location: foo Set-Cookie: SESSIONID=MaliciousValue", - }, - { - uri: `http://localhost:3000/?redirect=foo%0dSet-Cookie:%20SESSIONID=MaliciousValue%0d%0a`, - expectedRedirect: "foo\rSet-Cookie: SESSIONID=MaliciousValue\r\n", - expectedLocation: "Location: foo Set-Cookie: SESSIONID=MaliciousValue", - }, - { - uri: `http://localhost:3000/?redirect=foo%0aSet-Cookie:%20SESSIONID=MaliciousValue%0d%0a`, - expectedRedirect: "foo\nSet-Cookie: SESSIONID=MaliciousValue\r\n", - expectedLocation: "Location: foo Set-Cookie: SESSIONID=MaliciousValue", - }, - } - - for i, tcase := range testcases { - caseName := strconv.FormatInt(int64(i), 10) - t.Run(caseName, func(subT *testing.T) { - ctx := &RequestCtx{ - Request: Request{}, - Response: Response{}, - } - ctx.Request.SetRequestURI(tcase.uri) - - q := string(ctx.QueryArgs().Peek("redirect")) - if q != tcase.expectedRedirect { - subT.Errorf("unexpected redirect query value, got: %+v", q) - } - ctx.Response.Header.Set("Location", q) - - if !strings.Contains(ctx.Response.String(), tcase.expectedLocation) { - subT.Errorf("invalid escaping, got\n%q", ctx.Response.String()) - } - }) - } -} - -func TestRequestCopyTo(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var req Request - - // empty copy - testRequestCopyTo(t, &req) - - // init - expectedContentType := "application/x-www-form-urlencoded; charset=UTF-8" - expectedHost := "test.com" - expectedBody := "0123=56789" - s := fmt.Sprintf("POST / HTTP/1.1\r\nHost: %s\r\nContent-Type: %s\r\nContent-Length: %d\r\n\r\n%s", - expectedHost, expectedContentType, len(expectedBody), expectedBody) - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := req.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - testRequestCopyTo(t, &req) -} - -func TestResponseCopyTo(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var resp Response - - // empty copy - testResponseCopyTo(t, &resp) - - // init resp - resp.laddr = zeroTCPAddr - resp.SkipBody = true - resp.Header.SetStatusCode(200) - resp.SetBodyString("test") - testResponseCopyTo(t, &resp) -} - -func testRequestCopyTo(t *testing.T, src *Request) { - var dst Request - src.CopyTo(&dst) - - if !reflect.DeepEqual(*src, dst) { //nolint:govet - t.Fatalf("RequestCopyTo fail, src: \n%+v\ndst: \n%+v\n", *src, dst) //nolint:govet - } -} - -func testResponseCopyTo(t *testing.T, src *Response) { - var dst Response - src.CopyTo(&dst) - - if !reflect.DeepEqual(*src, dst) { //nolint:govet - t.Fatalf("ResponseCopyTo fail, src: \n%+v\ndst: \n%+v\n", *src, dst) //nolint:govet - } -} - -func TestRequestBodyStreamWithTrailer(t *testing.T) { - t.Parallel() - - testRequestBodyStreamWithTrailer(t, nil, false) - - body := createFixedBody(1e5) - testRequestBodyStreamWithTrailer(t, body, false) - testRequestBodyStreamWithTrailer(t, body, true) -} - -func testRequestBodyStreamWithTrailer(t *testing.T, body []byte, disableNormalizing bool) { - expectedTrailer := map[string]string{ - "foo": "testfoo", - "bar": "testbar", - } - - var req1 Request - req1.Header.disableNormalizing = disableNormalizing - req1.SetHost("google.com") - req1.SetBodyStream(bytes.NewBuffer(body), -1) - for k, v := range expectedTrailer { - err := req1.Header.AddTrailer(k) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - req1.Header.Set(k, v) - } - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := req1.Write(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var req2 Request - req2.Header.disableNormalizing = disableNormalizing - br := bufio.NewReader(w) - if err := req2.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - reqBody := req2.Body() - if !bytes.Equal(reqBody, body) { - t.Fatalf("unexpected body: %q. Expecting %q", reqBody, body) - } - - for k, v := range expectedTrailer { - kBytes := []byte(k) - normalizeHeaderKey(kBytes, disableNormalizing) - r := req2.Header.Peek(k) - if string(r) != v { - t.Fatalf("unexpected trailer header %q: %q. Expecting %q", kBytes, r, v) - } - } -} - -func TestResponseBodyStreamWithTrailer(t *testing.T) { - t.Parallel() - - testResponseBodyStreamWithTrailer(t, nil, false) - - body := createFixedBody(1e5) - testResponseBodyStreamWithTrailer(t, body, false) - testResponseBodyStreamWithTrailer(t, body, true) -} - -func testResponseBodyStreamWithTrailer(t *testing.T, body []byte, disableNormalizing bool) { - expectedTrailer := map[string]string{ - "foo": "testfoo", - "bar": "testbar", - } - var resp1 Response - resp1.Header.disableNormalizing = disableNormalizing - resp1.SetBodyStream(bytes.NewReader(body), -1) - for k, v := range expectedTrailer { - err := resp1.Header.AddTrailer(k) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - resp1.Header.Set(k, v) - } - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := resp1.Write(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var resp2 Response - resp2.Header.disableNormalizing = disableNormalizing - br := bufio.NewReader(w) - if err := resp2.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - respBody := resp2.Body() - if !bytes.Equal(respBody, body) { - t.Fatalf("unexpected body: %q. Expecting %q", respBody, body) - } - - for k, v := range expectedTrailer { - kBytes := []byte(k) - normalizeHeaderKey(kBytes, disableNormalizing) - r := resp2.Header.Peek(k) - if string(r) != v { - t.Fatalf("unexpected trailer header %q: %q. Expecting %q", kBytes, r, v) - } - } -} - -func TestResponseBodyStreamDeflate(t *testing.T) { - t.Parallel() - - body := createFixedBody(1e5) - - // Verifies https://infini.sh/framework/lib/fasthttp/issues/176 - // when Content-Length is explicitly set. - testResponseBodyStreamDeflate(t, body, len(body)) - - // Verifies that 'transfer-encoding: chunked' works as expected. - testResponseBodyStreamDeflate(t, body, -1) -} - -func TestResponseBodyStreamGzip(t *testing.T) { - t.Parallel() - - body := createFixedBody(1e5) - - // Verifies https://infini.sh/framework/lib/fasthttp/issues/176 - // when Content-Length is explicitly set. - testResponseBodyStreamGzip(t, body, len(body)) - - // Verifies that 'transfer-encoding: chunked' works as expected. - testResponseBodyStreamGzip(t, body, -1) -} - -func testResponseBodyStreamDeflate(t *testing.T, body []byte, bodySize int) { - var r Response - r.SetBodyStream(bytes.NewReader(body), bodySize) - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := r.WriteDeflate(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var resp Response - br := bufio.NewReader(w) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - respBody, err := resp.BodyInflate() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !bytes.Equal(respBody, body) { - t.Fatalf("unexpected body: %q. Expecting %q", respBody, body) - } - // check for invalid - resp.SetBody([]byte("invalid")) - _, errDeflate := resp.BodyInflate() - if errDeflate == nil || errDeflate.Error() != "zlib: invalid header" { - t.Fatalf("expected error: 'zlib: invalid header' but was %v", errDeflate) - } -} - -func testResponseBodyStreamGzip(t *testing.T, body []byte, bodySize int) { - var r Response - r.SetBodyStream(bytes.NewReader(body), bodySize) - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := r.WriteGzip(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var resp Response - br := bufio.NewReader(w) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - respBody, err := resp.BodyGunzip() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !bytes.Equal(respBody, body) { - t.Fatalf("unexpected body: %q. Expecting %q", respBody, body) - } - // check for invalid - resp.SetBody([]byte("invalid")) - _, errUnzip := resp.BodyGunzip() - if errUnzip == nil || errUnzip.Error() != "unexpected EOF" { - t.Fatalf("expected error: 'unexpected EOF' but was %v", errUnzip) - } -} - -func TestResponseWriteGzipNilBody(t *testing.T) { - t.Parallel() - - var r Response - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := r.WriteGzip(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestResponseWriteDeflateNilBody(t *testing.T) { - t.Parallel() - - var r Response - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := r.WriteDeflate(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestResponseBodyUncompressed(t *testing.T) { - body := "body" - var r Response - r.SetBodyStream(bytes.NewReader([]byte(body)), len(body)) - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := r.WriteDeflate(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var resp Response - br := bufio.NewReader(w) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - ce := resp.Header.ContentEncoding() - if string(ce) != "deflate" { - t.Fatalf("unexpected Content-Encoding: %s", ce) - } - respBody, err := resp.BodyUncompressed() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(respBody) != body { - t.Fatalf("unexpected body: %q. Expecting %q", respBody, body) - } - - // check for invalid encoding - resp.Header.SetContentEncoding("invalid") - _, decodeErr := resp.BodyUncompressed() - if decodeErr != ErrContentEncodingUnsupported { - t.Fatalf("unexpected error: %v", decodeErr) - } -} - -func TestResponseSwapBodySerial(t *testing.T) { - t.Parallel() - - testResponseSwapBody(t) -} - -func TestResponseSwapBodyConcurrent(t *testing.T) { - t.Parallel() - - ch := make(chan struct{}) - for i := 0; i < 10; i++ { - go func() { - testResponseSwapBody(t) - ch <- struct{}{} - }() - } - - for i := 0; i < 10; i++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } -} - -func testResponseSwapBody(t *testing.T) { - var b []byte - r := defaultHTTPPool.AcquireResponse() - for i := 0; i < 20; i++ { - bOrig := r.Body() - b = r.SwapBody(b) - if !bytes.Equal(bOrig, b) { - t.Fatalf("unexpected body returned: %q. Expecting %q", b, bOrig) - } - r.AppendBodyString("foobar") - } - - s := "aaaabbbbcccc" - b = b[:0] - for i := 0; i < 10; i++ { - r.SetBodyStream(bytes.NewBufferString(s), len(s)) - b = r.SwapBody(b) - if string(b) != s { - t.Fatalf("unexpected body returned: %q. Expecting %q", b, s) - } - b = r.SwapBody(b) - if len(b) > 0 { - t.Fatalf("unexpected body with non-zero size returned: %q", b) - } - } - defaultHTTPPool.ReleaseResponse(r) -} - -func TestRequestSwapBodySerial(t *testing.T) { - t.Parallel() - - testRequestSwapBody(t) -} - -func TestRequestSwapBodyConcurrent(t *testing.T) { - t.Parallel() - - ch := make(chan struct{}) - for i := 0; i < 10; i++ { - go func() { - testRequestSwapBody(t) - ch <- struct{}{} - }() - } - - for i := 0; i < 10; i++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } -} - -func testRequestSwapBody(t *testing.T) { - var b []byte - r := defaultHTTPPool.AcquireRequest() - for i := 0; i < 20; i++ { - bOrig := r.Body() - b = r.SwapBody(b) - if !bytes.Equal(bOrig, b) { - t.Fatalf("unexpected body returned: %q. Expecting %q", b, bOrig) - } - r.AppendBodyString("foobar") - } - - s := "aaaabbbbcccc" - b = b[:0] - for i := 0; i < 10; i++ { - r.SetBodyStream(bytes.NewBufferString(s), len(s)) - b = r.SwapBody(b) - if string(b) != s { - t.Fatalf("unexpected body returned: %q. Expecting %q", b, s) - } - b = r.SwapBody(b) - if len(b) > 0 { - t.Fatalf("unexpected body with non-zero size returned: %q", b) - } - } - defaultHTTPPool.ReleaseRequest(r) -} - -func TestRequestHostFromRequestURI(t *testing.T) { - t.Parallel() - - hExpected := "foobar.com" - var req Request - req.SetRequestURI("http://proxy-host:123/foobar?baz") - req.SetHost(hExpected) - h := req.Host() - if string(h) != hExpected { - t.Fatalf("unexpected host set: %q. Expecting %q", h, hExpected) - } -} - -func TestRequestHostFromHeader(t *testing.T) { - t.Parallel() - - hExpected := "foobar.com" - var req Request - req.Header.SetHost(hExpected) - h := req.Host() - if string(h) != hExpected { - t.Fatalf("unexpected host set: %q. Expecting %q", h, hExpected) - } -} - -func TestRequestContentTypeWithCharsetIssue100(t *testing.T) { - t.Parallel() - - expectedContentType := "application/x-www-form-urlencoded; charset=UTF-8" - expectedBody := "0123=56789" - s := fmt.Sprintf("POST / HTTP/1.1\r\nContent-Type: %s\r\nContent-Length: %d\r\n\r\n%s", - expectedContentType, len(expectedBody), expectedBody) - - br := bufio.NewReader(bytes.NewBufferString(s)) - var r Request - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - body := r.Body() - if string(body) != expectedBody { - t.Fatalf("unexpected body %q. Expecting %q", body, expectedBody) - } - ct := r.Header.ContentType() - if string(ct) != expectedContentType { - t.Fatalf("unexpected content-type %q. Expecting %q", ct, expectedContentType) - } - args := r.PostArgs() - if args.Len() != 1 { - t.Fatalf("unexpected number of POST args: %d. Expecting 1", args.Len()) - } - av := args.Peek("0123") - if string(av) != "56789" { - t.Fatalf("unexpected POST arg value: %q. Expecting %q", av, "56789") - } -} - -func TestRequestReadMultipartFormWithFile(t *testing.T) { - t.Parallel() - - s := `POST /upload HTTP/1.1 -Host: localhost:10000 -Content-Length: 521 -Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryJwfATyF8tmxSJnLg - -------WebKitFormBoundaryJwfATyF8tmxSJnLg -Content-Disposition: form-data; name="f1" - -value1 -------WebKitFormBoundaryJwfATyF8tmxSJnLg -Content-Disposition: form-data; name="fileaaa"; filename="TODO" -Content-Type: application/octet-stream - -- SessionClient with referer and cookies support. -- Client with requests' pipelining support. -- ProxyHandler similar to FSHandler. -- WebSockets. See https://tools.ietf.org/html/rfc6455 . -- HTTP/2.0. See https://tools.ietf.org/html/rfc7540 . - -------WebKitFormBoundaryJwfATyF8tmxSJnLg-- -tailfoobar` - - br := bufio.NewReader(bytes.NewBufferString(s)) - - var r Request - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - tail, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(tail) != "tailfoobar" { - t.Fatalf("unexpected tail %q. Expecting %q", tail, "tailfoobar") - } - - f, err := r.MultipartForm() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer r.RemoveMultipartFormFiles() - - // verify values - if len(f.Value) != 1 { - t.Fatalf("unexpected number of values in multipart form: %d. Expecting 1", len(f.Value)) - } - for k, vv := range f.Value { - if k != "f1" { - t.Fatalf("unexpected value name %q. Expecting %q", k, "f1") - } - if len(vv) != 1 { - t.Fatalf("unexpected number of values %d. Expecting 1", len(vv)) - } - v := vv[0] - if v != "value1" { - t.Fatalf("unexpected value %q. Expecting %q", v, "value1") - } - } - - // verify files - if len(f.File) != 1 { - t.Fatalf("unexpected number of file values in multipart form: %d. Expecting 1", len(f.File)) - } - for k, vv := range f.File { - if k != "fileaaa" { - t.Fatalf("unexpected file value name %q. Expecting %q", k, "fileaaa") - } - if len(vv) != 1 { - t.Fatalf("unexpected number of file values %d. Expecting 1", len(vv)) - } - v := vv[0] - if v.Filename != "TODO" { - t.Fatalf("unexpected filename %q. Expecting %q", v.Filename, "TODO") - } - ct := v.Header.Get("Content-Type") - if ct != "application/octet-stream" { - t.Fatalf("unexpected content-type %q. Expecting %q", ct, "application/octet-stream") - } - } -} - -func TestRequestSetURI(t *testing.T) { - t.Parallel() - - var r Request - - uri := "/foo/bar?baz" - u := &URI{} - u.Parse(nil, []byte(uri)) //nolint:errcheck - // Set request uri via SetURI() - r.SetURI(u) // copies URI - // modifying an original URI struct doesn't affect stored URI inside of request - u.SetPath("newPath") - if string(r.RequestURI()) != uri { - t.Fatalf("unexpected request uri %q. Expecting %q", r.RequestURI(), uri) - } - - // Set request uri to nil just resets the URI - r.Reset() - uri = "/" - r.SetURI(nil) - if string(r.RequestURI()) != uri { - t.Fatalf("unexpected request uri %q. Expecting %q", r.RequestURI(), uri) - } -} - -func TestRequestRequestURI(t *testing.T) { - t.Parallel() - - var r Request - - // Set request uri via SetRequestURI() - uri := "/foo/bar?baz" - r.SetRequestURI(uri) - if string(r.RequestURI()) != uri { - t.Fatalf("unexpected request uri %q. Expecting %q", r.RequestURI(), uri) - } - - // Set request uri via Request.URI().Update() - r.Reset() - uri = "/aa/bbb?ccc=sdfsdf" - r.PhantomURI().Update(uri) - if string(r.RequestURI()) != uri { - t.Fatalf("unexpected request uri %q. Expecting %q", r.RequestURI(), uri) - } - - // update query args in the request uri - qa := r.PhantomURI().QueryArgs() - qa.Reset() - qa.Set("foo", "bar") - uri = "/aa/bbb?foo=bar" - if string(r.RequestURI()) != uri { - t.Fatalf("unexpected request uri %q. Expecting %q", r.RequestURI(), uri) - } -} - -func TestRequestUpdateURI(t *testing.T) { - t.Parallel() - - var r Request - r.Header.SetHost("aaa.bbb") - r.SetRequestURI("/lkjkl/kjl") - - // Modify request uri and host via URI() object and make sure - // the requestURI and Host header are properly updated - u := r.PhantomURI() - u.SetPath("/123/432.html") - u.SetHost("foobar.com") - a := u.QueryArgs() - a.Set("aaa", "bcse") - - s := r.String() - if !strings.HasPrefix(s, "GET /123/432.html?aaa=bcse") { - t.Fatalf("cannot find %q in %q", "GET /123/432.html?aaa=bcse", s) - } - if !strings.Contains(s, "\r\nHost: foobar.com\r\n") { - t.Fatalf("cannot find %q in %q", "\r\nHost: foobar.com\r\n", s) - } -} - -func TestUseHostHeader(t *testing.T) { - t.Parallel() - - var r Request - r.UseHostHeader = true - r.Header.SetHost("aaa.bbb") - r.SetRequestURI("/lkjkl/kjl") - - // Modify request uri and host via URI() object and make sure - // the requestURI and Host header are properly updated - u := r.PhantomURI() - u.SetPath("/123/432.html") - u.SetHost("foobar.com") - a := u.QueryArgs() - a.Set("aaa", "bcse") - - s := r.String() - if !strings.HasPrefix(s, "GET /123/432.html?aaa=bcse") { - t.Fatalf("cannot find %q in %q", "GET /123/432.html?aaa=bcse", s) - } - if !strings.Contains(s, "\r\nHost: aaa.bbb\r\n") { - t.Fatalf("cannot find %q in %q", "\r\nHost: aaa.bbb\r\n", s) - } -} - -func TestUseHostHeader2(t *testing.T) { - t.Parallel() - testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Host != "SomeHost" { - http.Error(w, fmt.Sprintf("Expected Host header to be '%q', but got '%q'", "SomeHost", r.Host), http.StatusBadRequest) - } else { - w.WriteHeader(http.StatusOK) - } - })) - defer testServer.Close() - - client := &Client{} - req := defaultHTTPPool.AcquireRequest() - defer defaultHTTPPool.ReleaseRequest(req) - resp := defaultHTTPPool.AcquireResponse() - defer defaultHTTPPool.ReleaseResponse(resp) - - req.SetRequestURI(testServer.URL) - req.UseHostHeader = true - req.Header.SetHost("SomeHost") - if err := client.DoTimeout(req, resp, 1*time.Second); err != nil { - t.Fatalf("DoTimeout returned an error '%v'", err) - } else { - if resp.StatusCode() != http.StatusOK { - t.Fatalf("DoTimeout: %v", resp.body) - } - } - if err := client.Do(req, resp); err != nil { - t.Fatalf("DoTimeout returned an error '%v'", err) - } else { - if resp.StatusCode() != http.StatusOK { - t.Fatalf("Do: %q", resp.body) - } - } -} - -func TestUseHostHeaderAfterRelease(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - req := defaultHTTPPool.AcquireRequest() - req.UseHostHeader = true - defaultHTTPPool.ReleaseRequest(req) - - req = defaultHTTPPool.AcquireRequest() - defer defaultHTTPPool.ReleaseRequest(req) - if req.UseHostHeader { - t.Fatalf("UseHostHeader was not released in ReleaseRequest()") - } -} - -func TestRequestBodyStreamMultipleBodyCalls(t *testing.T) { - t.Parallel() - - var r Request - - s := "foobar baz abc" - if r.IsBodyStream() { - t.Fatalf("IsBodyStream must return false") - } - r.SetBodyStream(bytes.NewBufferString(s), len(s)) - if !r.IsBodyStream() { - t.Fatalf("IsBodyStream must return true") - } - for i := 0; i < 10; i++ { - body := r.Body() - if string(body) != s { - t.Fatalf("unexpected body %q. Expecting %q. iteration %d", body, s, i) - } - } -} - -func TestResponseBodyStreamMultipleBodyCalls(t *testing.T) { - t.Parallel() - - var r Response - - s := "foobar baz abc" - if r.IsBodyStream() { - t.Fatalf("IsBodyStream must return false") - } - r.SetBodyStream(bytes.NewBufferString(s), len(s)) - if !r.IsBodyStream() { - t.Fatalf("IsBodyStream must return true") - } - for i := 0; i < 10; i++ { - body := r.Body() - if string(body) != s { - t.Fatalf("unexpected body %q. Expecting %q. iteration %d", body, s, i) - } - } -} - -func TestRequestBodyWriteToPlain(t *testing.T) { - t.Parallel() - - var r Request - - expectedS := "foobarbaz" - r.AppendBodyString(expectedS) - - testBodyWriteTo(t, &r, expectedS, true) -} - -func TestResponseBodyWriteToPlain(t *testing.T) { - t.Parallel() - - var r Response - - expectedS := "foobarbaz" - r.AppendBodyString(expectedS) - - testBodyWriteTo(t, &r, expectedS, true) -} - -func TestResponseBodyWriteToStream(t *testing.T) { - t.Parallel() - - var r Response - - expectedS := "aaabbbccc" - buf := bytes.NewBufferString(expectedS) - if r.IsBodyStream() { - t.Fatalf("IsBodyStream must return false") - } - r.SetBodyStream(buf, len(expectedS)) - if !r.IsBodyStream() { - t.Fatalf("IsBodyStream must return true") - } - - testBodyWriteTo(t, &r, expectedS, false) -} - -func TestRequestBodyWriteToMultipart(t *testing.T) { - t.Parallel() - - expectedS := "--foobar\r\nContent-Disposition: form-data; name=\"key_0\"\r\n\r\nvalue_0\r\n--foobar--\r\n" - s := fmt.Sprintf("POST / HTTP/1.1\r\nHost: aaa\r\nContent-Type: multipart/form-data; boundary=foobar\r\nContent-Length: %d\r\n\r\n%s", - len(expectedS), expectedS) - - var r Request - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - testBodyWriteTo(t, &r, expectedS, true) -} - -type bodyWriterTo interface { - BodyWriteTo(io.Writer) error - Body() []byte -} - -func testBodyWriteTo(t *testing.T, bw bodyWriterTo, expectedS string, isRetainedBody bool) { - var buf bytebufferpool.ByteBuffer - if err := bw.BodyWriteTo(&buf); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - s := buf.B - if string(s) != expectedS { - t.Fatalf("unexpected result %q. Expecting %q", s, expectedS) - } - - body := bw.Body() - if isRetainedBody { - if string(body) != expectedS { - t.Fatalf("unexpected body %q. Expecting %q", body, expectedS) - } - } else { - if len(body) > 0 { - t.Fatalf("unexpected non-zero body after BodyWriteTo: %q", body) - } - } -} - -func TestRequestReadEOF(t *testing.T) { - t.Parallel() - - var r Request - - br := bufio.NewReader(&bytes.Buffer{}) - err := r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err != io.EOF { - t.Fatalf("unexpected error: %v. Expecting %v", err, io.EOF) - } - - // incomplete request mustn't return io.EOF - br = bufio.NewReader(bytes.NewBufferString("POST / HTTP/1.1\r\nContent-Type: aa\r\nContent-Length: 1234\r\n\r\nIncomplete body")) - err = r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err == io.EOF { - t.Fatalf("expecting non-EOF error") - } -} - -func TestResponseReadEOF(t *testing.T) { - t.Parallel() - - var r Response - - br := bufio.NewReader(&bytes.Buffer{}) - err := r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err != io.EOF { - t.Fatalf("unexpected error: %v. Expecting %v", err, io.EOF) - } - - // incomplete response mustn't return io.EOF - br = bufio.NewReader(bytes.NewBufferString("HTTP/1.1 200 OK\r\nContent-Type: aaa\r\nContent-Length: 123\r\n\r\nIncomplete body")) - err = r.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err == io.EOF { - t.Fatalf("expecting non-EOF error") - } -} - -func TestRequestReadNoBody(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var r Request - - br := bufio.NewReader(bytes.NewBufferString("GET / HTTP/1.1\r\n\r\n")) - err := r.Read(br) - r.SetHost("foobar") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - s := r.String() - if strings.Contains(s, "Content-Length: ") { - t.Fatalf("unexpected Content-Length") - } -} - -func TestResponseWriteTo(t *testing.T) { - t.Parallel() - - var r Response - - r.SetBodyString("foobar") - - s := r.String() - var buf bytebufferpool.ByteBuffer - n, err := r.WriteTo(&buf) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if n != int64(len(s)) { - t.Fatalf("unexpected response length %d. Expecting %d", n, len(s)) - } - if string(buf.B) != s { - t.Fatalf("unexpected response %q. Expecting %q", buf.B, s) - } -} - -func TestRequestWriteTo(t *testing.T) { - t.Parallel() - - var r Request - - r.SetRequestURI("http://foobar.com/aaa/bbb") - - s := r.String() - var buf bytebufferpool.ByteBuffer - n, err := r.WriteTo(&buf) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if n != int64(len(s)) { - t.Fatalf("unexpected request length %d. Expecting %d", n, len(s)) - } - if string(buf.B) != s { - t.Fatalf("unexpected request %q. Expecting %q", buf.B, s) - } -} - -func TestResponseSkipBody(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var r Response - - // set StatusNotModified - r.Header.SetStatusCode(StatusNotModified) - r.SetBodyString("foobar") - s := r.String() - if strings.Contains(s, "\r\n\r\nfoobar") { - t.Fatalf("unexpected non-zero body in response %q", s) - } - if strings.Contains(s, "Content-Length: ") { - t.Fatalf("unexpected content-length in response %q", s) - } - if strings.Contains(s, "Content-Type: ") { - t.Fatalf("unexpected content-type in response %q", s) - } - - // set StatusNoContent - r.Header.SetStatusCode(StatusNoContent) - r.SetBodyString("foobar") - s = r.String() - if strings.Contains(s, "\r\n\r\nfoobar") { - t.Fatalf("unexpected non-zero body in response %q", s) - } - if strings.Contains(s, "Content-Length: ") { - t.Fatalf("unexpected content-length in response %q", s) - } - if strings.Contains(s, "Content-Type: ") { - t.Fatalf("unexpected content-type in response %q", s) - } - - // set StatusNoContent with statusMessage - r.Header.SetStatusCode(StatusNoContent) - r.Header.SetStatusMessage([]byte("NC")) - r.SetBodyString("foobar") - s = r.String() - if strings.Contains(s, "\r\n\r\nfoobar") { - t.Fatalf("unexpected non-zero body in response %q", s) - } - if strings.Contains(s, "Content-Length: ") { - t.Fatalf("unexpected content-length in response %q", s) - } - if strings.Contains(s, "Content-Type: ") { - t.Fatalf("unexpected content-type in response %q", s) - } - if !strings.HasPrefix(s, "HTTP/1.1 204 NC\r\n") { - t.Fatalf("expecting non-default status line in response %q", s) - } - - // explicitly skip body - r.Header.SetStatusCode(StatusOK) - r.SkipBody = true - r.SetBodyString("foobar") - s = r.String() - if strings.Contains(s, "\r\n\r\nfoobar") { - t.Fatalf("unexpected non-zero body in response %q", s) - } - if !strings.Contains(s, "Content-Length: 6\r\n") { - t.Fatalf("expecting content-length in response %q", s) - } - if !strings.Contains(s, "Content-Type: ") { - t.Fatalf("expecting content-type in response %q", s) - } -} - -func TestRequestNoContentLength(t *testing.T) { - t.Parallel() - - var r Request - - r.Header.SetMethod(MethodHead) - r.Header.SetHost("foobar") - - s := r.String() - if strings.Contains(s, "Content-Length: ") { - t.Fatalf("unexpected content-length in HEAD request %q", s) - } - - r.Header.SetMethod(MethodPost) - fmt.Fprintf(r.BodyWriter(), "foobar body") - s = r.String() - if !strings.Contains(s, "Content-Length: ") { - t.Fatalf("missing content-length header in non-GET request %q", s) - } -} - -func TestRequestReadGzippedBody(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var r Request - - bodyOriginal := "foo bar baz compress me better!" - body := AppendGzipBytes(nil, []byte(bodyOriginal)) - s := fmt.Sprintf("POST /foobar HTTP/1.1\r\nContent-Type: foo/bar\r\nContent-Encoding: gzip\r\nContent-Length: %d\r\n\r\n%s", - len(body), body) - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if string(r.Header.ContentEncoding()) != "gzip" { - t.Fatalf("unexpected content-encoding: %q. Expecting %q", r.Header.ContentEncoding(), "gzip") - } - if r.Header.ContentLength() != len(body) { - t.Fatalf("unexpected content-length: %d. Expecting %d", r.Header.ContentLength(), len(body)) - } - if string(r.Body()) != string(body) { - t.Fatalf("unexpected body: %q. Expecting %q", r.Body(), body) - } - - bodyGunzipped, err := AppendGunzipBytes(nil, r.Body()) - if err != nil { - t.Fatalf("unexpected error when uncompressing data: %v", err) - } - if string(bodyGunzipped) != bodyOriginal { - t.Fatalf("unexpected uncompressed body %q. Expecting %q", bodyGunzipped, bodyOriginal) - } -} - -func TestRequestReadPostNoBody(t *testing.T) { - t.Parallel() - - var r Request - - s := "POST /foo/bar HTTP/1.1\r\nContent-Type: aaa/bbb\r\n\r\naaaa" - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if string(r.Header.RequestURI()) != "/foo/bar" { - t.Fatalf("unexpected request uri %q. Expecting %q", r.Header.RequestURI(), "/foo/bar") - } - if string(r.Header.ContentType()) != "aaa/bbb" { - t.Fatalf("unexpected content-type %q. Expecting %q", r.Header.ContentType(), "aaa/bbb") - } - if len(r.Body()) != 0 { - t.Fatalf("unexpected body found %q. Expecting empty body", r.Body()) - } - if r.Header.ContentLength() != 0 { - t.Fatalf("unexpected content-length: %d. Expecting 0", r.Header.ContentLength()) - } - - tail, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(tail) != "aaaa" { - t.Fatalf("unexpected tail %q. Expecting %q", tail, "aaaa") - } -} - -func TestRequestContinueReadBody(t *testing.T) { - t.Parallel() - - s := "PUT /foo/bar HTTP/1.1\r\nExpect: 100-continue\r\nContent-Length: 5\r\nContent-Type: foo/bar\r\n\r\nabcdef4343" - br := bufio.NewReader(bytes.NewBufferString(s)) - - var r Request - if err := r.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !r.MayContinue() { - t.Fatalf("MayContinue must return true") - } - - if err := r.ContinueReadBody(br, 0, true); err != nil { - t.Fatalf("error when reading request body: %v", err) - } - body := r.Body() - if string(body) != "abcde" { - t.Fatalf("unexpected body %q. Expecting %q", body, "abcde") - } - - tail, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(tail) != "f4343" { - t.Fatalf("unexpected tail %q. Expecting %q", tail, "f4343") - } -} - -func TestRequestContinueReadBodyDisablePrereadMultipartForm(t *testing.T) { - t.Parallel() - - var w bytes.Buffer - mw := multipart.NewWriter(&w) - for i := 0; i < 10; i++ { - k := fmt.Sprintf("key_%d", i) - v := fmt.Sprintf("value_%d", i) - if err := mw.WriteField(k, v); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - boundary := mw.Boundary() - if err := mw.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - formData := w.Bytes() - - s := fmt.Sprintf("POST / HTTP/1.1\r\nHost: aaa\r\nContent-Type: multipart/form-data; boundary=%s\r\nContent-Length: %d\r\n\r\n%s", - boundary, len(formData), formData) - br := bufio.NewReader(bytes.NewBufferString(s)) - - var r Request - - if err := r.Header.Read(br); err != nil { - t.Fatalf("unexpected error reading headers: %v", err) - } - - if err := r.readLimitBody(br, 10000, false, false); err != nil { - t.Fatalf("unexpected error reading body: %v", err) - } - - if r.multipartForm != nil { - t.Fatalf("The multipartForm of the Request must be nil") - } - - if string(formData) != string(r.Body()) { - t.Fatalf("The body given must equal the body in the Request") - } -} - -func TestRequestMayContinue(t *testing.T) { - t.Parallel() - - var r Request - if r.MayContinue() { - t.Fatalf("MayContinue on empty request must return false") - } - - r.Header.Set("Expect", "123sdfds") - if r.MayContinue() { - t.Fatalf("MayContinue on invalid Expect header must return false") - } - - r.Header.Set("Expect", "100-continue") - if !r.MayContinue() { - t.Fatalf("MayContinue on 'Expect: 100-continue' header must return true") - } -} - -func TestResponseGzipStream(t *testing.T) { - t.Parallel() - - var r Response - if r.IsBodyStream() { - t.Fatalf("IsBodyStream must return false") - } - r.SetBodyStreamWriter(func(w *bufio.Writer) { - fmt.Fprintf(w, "foo") - w.Flush() - time.Sleep(time.Millisecond) - w.Write([]byte("barbaz")) //nolint:errcheck - w.Flush() //nolint:errcheck - time.Sleep(time.Millisecond) - fmt.Fprintf(w, "1234") //nolint:errcheck - if err := w.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - if !r.IsBodyStream() { - t.Fatalf("IsBodyStream must return true") - } - testResponseGzipExt(t, &r, "foobarbaz1234") -} - -func TestResponseDeflateStream(t *testing.T) { - t.Parallel() - - var r Response - if r.IsBodyStream() { - t.Fatalf("IsBodyStream must return false") - } - r.SetBodyStreamWriter(func(w *bufio.Writer) { - w.Write([]byte("foo")) //nolint:errcheck - w.Flush() //nolint:errcheck - fmt.Fprintf(w, "barbaz") //nolint:errcheck - w.Flush() //nolint:errcheck - w.Write([]byte("1234")) //nolint:errcheck - if err := w.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - }) - if !r.IsBodyStream() { - t.Fatalf("IsBodyStream must return true") - } - testResponseDeflateExt(t, &r, "foobarbaz1234") -} - -func TestResponseDeflate(t *testing.T) { - t.Parallel() - - for _, s := range compressTestcases { - testResponseDeflate(t, s) - } -} - -func TestResponseGzip(t *testing.T) { - t.Parallel() - - for _, s := range compressTestcases { - testResponseGzip(t, s) - } -} - -func testResponseDeflate(t *testing.T, s string) { - var r Response - r.SetBodyString(s) - testResponseDeflateExt(t, &r, s) - - // make sure the uncompressible Content-Type isn't compressed - r.Reset() - r.Header.SetContentType("image/jpeg") - r.SetBodyString(s) - testResponseDeflateExt(t, &r, s) -} - -func testResponseDeflateExt(t *testing.T, r *Response, s string) { - isCompressible := isCompressibleResponse(r, s) - - var buf bytes.Buffer - var err error - bw := bufio.NewWriter(&buf) - if err = r.WriteDeflate(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err = bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var r1 Response - br := bufio.NewReader(&buf) - if err = r1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - ce := r1.Header.ContentEncoding() - var body []byte - if isCompressible { - if string(ce) != "deflate" { - t.Fatalf("unexpected Content-Encoding %q. Expecting %q. len(s)=%d, Content-Type: %q", - ce, "deflate", len(s), r.Header.ContentType()) - } - body, err = r1.BodyInflate() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - } else { - if len(ce) > 0 { - t.Fatalf("expecting empty Content-Encoding. Got %q", ce) - } - body = r1.Body() - } - if string(body) != s { - t.Fatalf("unexpected body %q. Expecting %q", body, s) - } -} - -func testResponseGzip(t *testing.T, s string) { - var r Response - r.SetBodyString(s) - testResponseGzipExt(t, &r, s) - - // make sure the uncompressible Content-Type isn't compressed - r.Reset() - r.Header.SetContentType("image/jpeg") - r.SetBodyString(s) - testResponseGzipExt(t, &r, s) -} - -func testResponseGzipExt(t *testing.T, r *Response, s string) { - isCompressible := isCompressibleResponse(r, s) - - var buf bytes.Buffer - var err error - bw := bufio.NewWriter(&buf) - if err = r.WriteGzip(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err = bw.Flush(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var r1 Response - br := bufio.NewReader(&buf) - if err = r1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - ce := r1.Header.ContentEncoding() - var body []byte - if isCompressible { - if string(ce) != "gzip" { - t.Fatalf("unexpected Content-Encoding %q. Expecting %q. len(s)=%d, Content-Type: %q", - ce, "gzip", len(s), r.Header.ContentType()) - } - body, err = r1.BodyGunzip() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - } else { - if len(ce) > 0 { - t.Fatalf("Expecting empty Content-Encoding. Got %q", ce) - } - body = r1.Body() - } - if string(body) != s { - t.Fatalf("unexpected body %q. Expecting %q", body, s) - } -} - -func isCompressibleResponse(r *Response, s string) bool { - isCompressible := r.Header.isCompressibleContentType() - if isCompressible && len(s) < minCompressLen && !r.IsBodyStream() { - isCompressible = false - } - return isCompressible -} - -func TestRequestMultipartForm(t *testing.T) { - t.Parallel() - - var w bytes.Buffer - mw := multipart.NewWriter(&w) - for i := 0; i < 10; i++ { - k := fmt.Sprintf("key_%d", i) - v := fmt.Sprintf("value_%d", i) - if err := mw.WriteField(k, v); err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - boundary := mw.Boundary() - if err := mw.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - formData := w.Bytes() - for i := 0; i < 5; i++ { - formData = testRequestMultipartForm(t, boundary, formData, 10) - } - - // verify request unmarshalling / marshalling - s := "POST / HTTP/1.1\r\nHost: aaa\r\nContent-Type: multipart/form-data; boundary=foobar\r\nContent-Length: 213\r\n\r\n--foobar\r\nContent-Disposition: form-data; name=\"key_0\"\r\n\r\nvalue_0\r\n--foobar\r\nContent-Disposition: form-data; name=\"key_1\"\r\n\r\nvalue_1\r\n--foobar\r\nContent-Disposition: form-data; name=\"key_2\"\r\n\r\nvalue_2\r\n--foobar--\r\n" - - var req Request - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := req.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - s = req.String() - br = bufio.NewReader(bytes.NewBufferString(s)) - if err := req.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - testRequestMultipartForm(t, "foobar", req.Body(), 3) -} - -func testRequestMultipartForm(t *testing.T, boundary string, formData []byte, partsCount int) []byte { - s := fmt.Sprintf("POST / HTTP/1.1\r\nHost: aaa\r\nContent-Type: multipart/form-data; boundary=%s\r\nContent-Length: %d\r\n\r\n%s", - boundary, len(formData), formData) - - var req Request - - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - if err := req.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - f, err := req.MultipartForm() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer req.RemoveMultipartFormFiles() - - if len(f.File) > 0 { - t.Fatalf("unexpected files found in the multipart form: %d", len(f.File)) - } - - if len(f.Value) != partsCount { - t.Fatalf("unexpected number of values found: %d. Expecting %d", len(f.Value), partsCount) - } - - for k, vv := range f.Value { - if len(vv) != 1 { - t.Fatalf("unexpected number of values found for key=%q: %d. Expecting 1", k, len(vv)) - } - if !strings.HasPrefix(k, "key_") { - t.Fatalf("unexpected key prefix=%q. Expecting %q", k, "key_") - } - v := vv[0] - if !strings.HasPrefix(v, "value_") { - t.Fatalf("unexpected value prefix=%q. expecting %q", v, "value_") - } - if k[len("key_"):] != v[len("value_"):] { - t.Fatalf("key and value suffixes don't match: %q vs %q", k, v) - } - } - - return req.Body() -} - -func TestResponseReadLimitBody(t *testing.T) { - t.Parallel() - - // response with content-length - testResponseReadLimitBodySuccess(t, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nContent-Length: 10\r\n\r\n9876543210", 10) - testResponseReadLimitBodySuccess(t, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nContent-Length: 10\r\n\r\n9876543210", 100) - testResponseReadLimitBodyError(t, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nContent-Length: 10\r\n\r\n9876543210", 9, ErrBodyTooLarge) - - // chunked response - testResponseReadLimitBodySuccess(t, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\n\r\n", 9) - testResponseReadLimitBodySuccess(t, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\nFoo: bar\r\n\r\n", 9) - testResponseReadLimitBodySuccess(t, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\n\r\n", 100) - testResponseReadLimitBodySuccess(t, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\nfoobar\r\n\r\n", 100) - testResponseReadLimitBodyError(t, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\n\r\n", 2, ErrBodyTooLarge) - - // identity response - testResponseReadLimitBodySuccess(t, "HTTP/1.1 400 OK\r\nContent-Type: aa\r\n\r\n123456", 6) - testResponseReadLimitBodySuccess(t, "HTTP/1.1 400 OK\r\nContent-Type: aa\r\n\r\n123456", 106) - testResponseReadLimitBodyError(t, "HTTP/1.1 400 OK\r\nContent-Type: aa\r\n\r\n123456", 5, ErrBodyTooLarge) -} - -func TestRequestReadLimitBody(t *testing.T) { - t.Parallel() - - // request with content-length - testRequestReadLimitBodySuccess(t, "POST /foo HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: 9\r\nContent-Type: aaa\r\n\r\n123456789", 9) - testRequestReadLimitBodySuccess(t, "POST /foo HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: 9\r\nContent-Type: aaa\r\n\r\n123456789", 92) - testRequestReadLimitBodyError(t, "POST /foo HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: 9\r\nContent-Type: aaa\r\n\r\n123456789", 5, ErrBodyTooLarge) - - // chunked request - testRequestReadLimitBodySuccess(t, "POST /a HTTP/1.1\r\nHost: a.com\r\nTransfer-Encoding: chunked\r\nContent-Type: aa\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\n\r\n", 9) - testRequestReadLimitBodySuccess(t, "POST /a HTTP/1.1\nHost: a.com\nTransfer-Encoding: chunked\nContent-Type: aa\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\nFoo: bar\r\n\r\n", 9) - testRequestReadLimitBodySuccess(t, "POST /a HTTP/1.1\r\nHost: a.com\r\nTransfer-Encoding: chunked\r\nContent-Type: aa\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\n\r\n", 999) - testRequestReadLimitBodySuccess(t, "POST /a HTTP/1.1\r\nHost: a.com\r\nTransfer-Encoding: chunked\r\nContent-Type: aa\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\nfoobar\r\n\r\n", 999) - testRequestReadLimitBodyError(t, "POST /a HTTP/1.1\r\nHost: a.com\r\nTransfer-Encoding: chunked\r\nContent-Type: aa\r\n\r\n6\r\nfoobar\r\n3\r\nbaz\r\n0\r\n\r\n", 8, ErrBodyTooLarge) -} - -func testResponseReadLimitBodyError(t *testing.T, s string, maxBodySize int, expectedErr error) { - var req Response - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - err := req.ReadLimitBody(br, maxBodySize) - if err == nil { - t.Fatalf("expecting error. s=%q, maxBodySize=%d", s, maxBodySize) - } - if err != expectedErr { - t.Fatalf("unexpected error: %v. Expecting %v. s=%q, maxBodySize=%d", err, expectedErr, s, maxBodySize) - } -} - -func testResponseReadLimitBodySuccess(t *testing.T, s string, maxBodySize int) { - var req Response - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - if err := req.ReadLimitBody(br, maxBodySize); err != nil { - t.Fatalf("unexpected error: %v. s=%q, maxBodySize=%d", err, s, maxBodySize) - } -} - -func testRequestReadLimitBodyError(t *testing.T, s string, maxBodySize int, expectedErr error) { - var req Request - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - err := req.ReadLimitBody(br, maxBodySize) - if err == nil { - t.Fatalf("expecting error. s=%q, maxBodySize=%d", s, maxBodySize) - } - if err != expectedErr { - t.Fatalf("unexpected error: %v. Expecting %v. s=%q, maxBodySize=%d", err, expectedErr, s, maxBodySize) - } -} - -func testRequestReadLimitBodySuccess(t *testing.T, s string, maxBodySize int) { - var req Request - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - if err := req.ReadLimitBody(br, maxBodySize); err != nil { - t.Fatalf("unexpected error: %v. s=%q, maxBodySize=%d", err, s, maxBodySize) - } -} - -func TestRequestString(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var r Request - r.SetRequestURI("http://foobar.com/aaa") - s := r.String() - expectedS := "GET /aaa HTTP/1.1\r\nHost: foobar.com\r\n\r\n" - if s != expectedS { - t.Fatalf("unexpected request: %q. Expecting %q", s, expectedS) - } -} - -func TestRequestBodyWriter(t *testing.T) { - var r Request - w := r.BodyWriter() - for i := 0; i < 10; i++ { - fmt.Fprintf(w, "%d", i) - } - if string(r.Body()) != "0123456789" { - t.Fatalf("unexpected body %q. Expecting %q", r.Body(), "0123456789") - } -} - -func TestResponseBodyWriter(t *testing.T) { - t.Parallel() - - var r Response - w := r.BodyWriter() - for i := 0; i < 10; i++ { - fmt.Fprintf(w, "%d", i) - } - if string(r.Body()) != "0123456789" { - t.Fatalf("unexpected body %q. Expecting %q", r.Body(), "0123456789") - } -} - -func TestRequestWriteRequestURINoHost(t *testing.T) { - t.Parallel() - - var req Request - req.Header.SetRequestURI("http://google.com/foo/bar?baz=aaa") - var w bytes.Buffer - bw := bufio.NewWriter(&w) - if err := req.Write(bw); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexepcted error: %v", err) - } - - var req1 Request - br := bufio.NewReader(&w) - if err := req1.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(req1.Header.Host()) != "google.com" { - t.Fatalf("unexpected host: %q. Expecting %q", req1.Header.Host(), "google.com") - } - if string(req.Header.RequestURI()) != "/foo/bar?baz=aaa" { - t.Fatalf("unexpected requestURI: %q. Expecting %q", req.Header.RequestURI(), "/foo/bar?baz=aaa") - } - - // verify that Request.Write returns error on non-absolute RequestURI - req.Reset() - req.Header.SetRequestURI("/foo/bar") - w.Reset() - bw.Reset(&w) - if err := req.Write(bw); err == nil { - t.Fatalf("expecting error") - } -} - -func TestSetRequestBodyStreamFixedSize(t *testing.T) { - t.Parallel() - - testSetRequestBodyStream(t, "a") - testSetRequestBodyStream(t, string(createFixedBody(4097))) - testSetRequestBodyStream(t, string(createFixedBody(100500))) -} - -func TestSetResponseBodyStreamFixedSize(t *testing.T) { - t.Parallel() - - testSetResponseBodyStream(t, "a") - testSetResponseBodyStream(t, string(createFixedBody(4097))) - testSetResponseBodyStream(t, string(createFixedBody(100500))) -} - -func TestSetRequestBodyStreamChunked(t *testing.T) { - t.Parallel() - - testSetRequestBodyStreamChunked(t, "", map[string]string{"Foo": "bar"}) - - body := "foobar baz aaa bbb ccc" - testSetRequestBodyStreamChunked(t, body, nil) - - body = string(createFixedBody(10001)) - testSetRequestBodyStreamChunked(t, body, map[string]string{"Foo": "test", "Bar": "test"}) -} - -func TestSetResponseBodyStreamChunked(t *testing.T) { - t.Parallel() - - testSetResponseBodyStreamChunked(t, "", map[string]string{"Foo": "bar"}) - - body := "foobar baz aaa bbb ccc" - testSetResponseBodyStreamChunked(t, body, nil) - - body = string(createFixedBody(10001)) - testSetResponseBodyStreamChunked(t, body, map[string]string{"Foo": "test", "Bar": "test"}) -} - -func testSetRequestBodyStream(t *testing.T, body string) { - var req Request - req.Header.SetHost("foobar.com") - req.Header.SetMethod(MethodPost) - - bodySize := len(body) - if req.IsBodyStream() { - t.Fatalf("IsBodyStream must return false") - } - req.SetBodyStream(bytes.NewBufferString(body), bodySize) - if !req.IsBodyStream() { - t.Fatalf("IsBodyStream must return true") - } - - var w bytes.Buffer - bw := bufio.NewWriter(&w) - if err := req.Write(bw); err != nil { - t.Fatalf("unexpected error when writing request: %v. body=%q", err, body) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error when flushing request: %v. body=%q", err, body) - } - - var req1 Request - br := bufio.NewReader(&w) - if err := req1.Read(br); err != nil { - t.Fatalf("unexpected error when reading request: %v. body=%q", err, body) - } - if string(req1.Body()) != body { - t.Fatalf("unexpected body %q. Expecting %q", req1.Body(), body) - } -} - -func testSetRequestBodyStreamChunked(t *testing.T, body string, trailer map[string]string) { - var req Request - req.Header.SetHost("foobar.com") - req.Header.SetMethod(MethodPost) - - if req.IsBodyStream() { - t.Fatalf("IsBodyStream must return false") - } - req.SetBodyStream(bytes.NewBufferString(body), -1) - if !req.IsBodyStream() { - t.Fatalf("IsBodyStream must return true") - } - - var w bytes.Buffer - bw := bufio.NewWriter(&w) - for k := range trailer { - err := req.Header.AddTrailer(k) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if err := req.Write(bw); err != nil { - t.Fatalf("unexpected error when writing request: %v. body=%q", err, body) - } - for k, v := range trailer { - req.Header.Set(k, v) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error when flushing request: %v. body=%q", err, body) - } - - var req1 Request - br := bufio.NewReader(&w) - if err := req1.Read(br); err != nil { - t.Fatalf("unexpected error when reading request: %v. body=%q", err, body) - } - if string(req1.Body()) != body { - t.Fatalf("unexpected body %q. Expecting %q", req1.Body(), body) - } - for k, v := range trailer { - r := req.Header.Peek(k) - if string(r) != v { - t.Fatalf("unexpected trailer %q. Expecting %q. Got %q", k, v, r) - } - } -} - -func testSetResponseBodyStream(t *testing.T, body string) { - var resp Response - bodySize := len(body) - if resp.IsBodyStream() { - t.Fatalf("IsBodyStream must return false") - } - resp.SetBodyStream(bytes.NewBufferString(body), bodySize) - if !resp.IsBodyStream() { - t.Fatalf("IsBodyStream must return true") - } - - var w bytes.Buffer - bw := bufio.NewWriter(&w) - if err := resp.Write(bw); err != nil { - t.Fatalf("unexpected error when writing response: %v. body=%q", err, body) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error when flushing response: %v. body=%q", err, body) - } - - var resp1 Response - br := bufio.NewReader(&w) - if err := resp1.Read(br); err != nil { - t.Fatalf("unexpected error when reading response: %v. body=%q", err, body) - } - if string(resp1.Body()) != body { - t.Fatalf("unexpected body %q. Expecting %q", resp1.Body(), body) - } -} - -func testSetResponseBodyStreamChunked(t *testing.T, body string, trailer map[string]string) { - var resp Response - if resp.IsBodyStream() { - t.Fatalf("IsBodyStream must return false") - } - resp.SetBodyStream(bytes.NewBufferString(body), -1) - if !resp.IsBodyStream() { - t.Fatalf("IsBodyStream must return true") - } - - var w bytes.Buffer - bw := bufio.NewWriter(&w) - for k := range trailer { - err := resp.Header.AddTrailer(k) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - } - if err := resp.Write(bw); err != nil { - t.Fatalf("unexpected error when writing response: %v. body=%q", err, body) - } - if err := bw.Flush(); err != nil { - t.Fatalf("unexpected error when flushing response: %v. body=%q", err, body) - } - for k, v := range trailer { - resp.Header.Set(k, v) - } - - var resp1 Response - br := bufio.NewReader(&w) - if err := resp1.Read(br); err != nil { - t.Fatalf("unexpected error when reading response: %v. body=%q", err, body) - } - if string(resp1.Body()) != body { - t.Fatalf("unexpected body %q. Expecting %q", resp1.Body(), body) - } - for k, v := range trailer { - r := resp.Header.Peek(k) - if string(r) != v { - t.Fatalf("unexpected trailer %q. Expecting %q. Got %q", k, v, r) - } - } -} - -func TestRound2(t *testing.T) { - t.Parallel() - - testRound2(t, 0, 0) - testRound2(t, 1, 1) - testRound2(t, 2, 2) - testRound2(t, 3, 4) - testRound2(t, 4, 4) - testRound2(t, 5, 8) - testRound2(t, 7, 8) - testRound2(t, 8, 8) - testRound2(t, 9, 16) - testRound2(t, 0x10001, 0x20000) - testRound2(t, math.MaxInt32-1, math.MaxInt32) -} - -func testRound2(t *testing.T, n, expectedRound2 int) { - if round2(n) != expectedRound2 { - t.Fatalf("Unexpected round2(%d)=%d. Expected %d", n, round2(n), expectedRound2) - } -} - -func TestRequestReadChunked(t *testing.T) { - t.Parallel() - - var req Request - - s := "POST /foo HTTP/1.1\r\nHost: google.com\r\nTransfer-Encoding: chunked\r\nContent-Type: aa/bb\r\n\r\n3\r\nabc\r\n5\r\n12345\r\n0\r\n\r\nTrail: test\r\n\r\n" - r := bytes.NewBufferString(s) - rb := bufio.NewReader(r) - err := req.Read(rb) - if err != nil { - t.Fatalf("Unexpected error when reading chunked request: %v", err) - } - expectedBody := "abc12345" - if string(req.Body()) != expectedBody { - t.Fatalf("Unexpected body %q. Expected %q", req.Body(), expectedBody) - } - verifyRequestHeader(t, &req.Header, -1, "/foo", "google.com", "", "aa/bb") - verifyTrailer(t, rb, map[string]string{"Trail": "test"}, true) -} - -// See: https://github.com/erikdubbelboer/fasthttp/issues/34 -func TestRequestChunkedWhitespace(t *testing.T) { - t.Parallel() - - var req Request - - s := "POST /foo HTTP/1.1\r\nHost: google.com\r\nTransfer-Encoding: chunked\r\nContent-Type: aa/bb\r\n\r\n3 \r\nabc\r\n0\r\n\r\n" - r := bytes.NewBufferString(s) - rb := bufio.NewReader(r) - err := req.Read(rb) - if err != nil { - t.Fatalf("Unexpected error when reading chunked request: %v", err) - } - expectedBody := "abc" - if string(req.Body()) != expectedBody { - t.Fatalf("Unexpected body %q. Expected %q", req.Body(), expectedBody) - } -} - -func TestResponseReadWithoutBody(t *testing.T) { - t.Parallel() - - var resp Response - - testResponseReadWithoutBody(t, &resp, "HTTP/1.1 304 Not Modified\r\nContent-Type: aa\r\nContent-Length: 1235\r\n\r\n", false, - 304, 1235, "aa") - - testResponseReadWithoutBody(t, &resp, "HTTP/1.1 204 Foo Bar\r\nContent-Type: aab\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n", false, - 204, -1, "aab") - - testResponseReadWithoutBody(t, &resp, "HTTP/1.1 123 AAA\r\nContent-Type: xxx\r\nContent-Length: 3434\r\n\r\n", false, - 123, 3434, "xxx") - - testResponseReadWithoutBody(t, &resp, "HTTP 200 OK\r\nContent-Type: text/xml\r\nContent-Length: 123\r\n\r\nfoobar\r\n", true, - 200, 123, "text/xml") - - // '100 Continue' must be skipped. - testResponseReadWithoutBody(t, &resp, "HTTP/1.1 100 Continue\r\nFoo-bar: baz\r\n\r\nHTTP/1.1 329 aaa\r\nContent-Type: qwe\r\nContent-Length: 894\r\n\r\n", true, - 329, 894, "qwe") -} - -func testResponseReadWithoutBody(t *testing.T, resp *Response, s string, skipBody bool, - expectedStatusCode, expectedContentLength int, expectedContentType string, -) { - t.Helper() - - r := bytes.NewBufferString(s) - rb := bufio.NewReader(r) - resp.SkipBody = skipBody - err := resp.Read(rb) - if err != nil { - t.Fatalf("Unexpected error when reading response without body: %v. response=%q", err, s) - } - if len(resp.Body()) != 0 { - t.Fatalf("Unexpected response body %q. Expected %q. response=%q", resp.Body(), "", s) - } - verifyResponseHeader(t, &resp.Header, expectedStatusCode, expectedContentLength, expectedContentType, "") - - // verify that ordinal response is read after null-body response - resp.SkipBody = false - testResponseReadSuccess(t, resp, "HTTP/1.1 300 OK\r\nContent-Length: 5\r\nContent-Type: bar\r\n\r\n56789aaa", - 300, 5, "bar", "56789", nil) -} - -func TestRequestSuccess(t *testing.T) { - t.Parallel() - - // empty method, user-agent and body - testRequestSuccess(t, "", "/foo/bar", "google.com", "", "", MethodGet) - - // non-empty user-agent - testRequestSuccess(t, MethodGet, "/foo/bar", "google.com", "MSIE", "", MethodGet) - - // non-empty method - testRequestSuccess(t, MethodHead, "/aaa", "fobar", "", "", MethodHead) - - // POST method with body - testRequestSuccess(t, MethodPost, "/bbb", "aaa.com", "Chrome aaa", "post body", MethodPost) - - // PUT method with body - testRequestSuccess(t, MethodPut, "/aa/bb", "a.com", "ome aaa", "put body", MethodPut) - - // only host is set - testRequestSuccess(t, "", "", "gooble.com", "", "", MethodGet) - - // get with body - testRequestSuccess(t, MethodGet, "/foo/bar", "aaa.com", "", "foobar", MethodGet) -} - -func TestResponseSuccess(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - // 200 response - testResponseSuccess(t, 200, "test/plain", "server", "foobar", - 200, "test/plain", "server") - - // response with missing statusCode - testResponseSuccess(t, 0, "text/plain", "server", "foobar", - 200, "text/plain", "server") - - // response with missing server - testResponseSuccess(t, 500, "aaa", "", "aaadfsd", - 500, "aaa", "") - - // empty body - testResponseSuccess(t, 200, "bbb", "qwer", "", - 200, "bbb", "qwer") - - // missing content-type - testResponseSuccess(t, 200, "", "asdfsd", "asdf", - 200, string(defaultContentType), "asdfsd") -} - -func testResponseSuccess(t *testing.T, statusCode int, contentType, serverName, body string, - expectedStatusCode int, expectedContentType, expectedServerName string) { - var resp Response - resp.SetStatusCode(statusCode) - resp.Header.Set("Content-Type", contentType) - resp.Header.Set("Server", serverName) - resp.SetBody([]byte(body)) - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - err := resp.Write(bw) - if err != nil { - t.Fatalf("Unexpected error when calling Response.Write(): %v", err) - } - if err = bw.Flush(); err != nil { - t.Fatalf("Unexpected error when flushing bufio.Writer: %v", err) - } - - var resp1 Response - br := bufio.NewReader(w) - if err = resp1.Read(br); err != nil { - t.Fatalf("Unexpected error when calling Response.Read(): %v", err) - } - if resp1.StatusCode() != expectedStatusCode { - t.Fatalf("Unexpected status code: %d. Expected %d", resp1.StatusCode(), expectedStatusCode) - } - if resp1.Header.ContentLength() != len(body) { - t.Fatalf("Unexpected content-length: %d. Expected %d", resp1.Header.ContentLength(), len(body)) - } - if string(resp1.Header.Peek(HeaderContentType)) != expectedContentType { - t.Fatalf("Unexpected content-type: %q. Expected %q", resp1.Header.Peek(HeaderContentType), expectedContentType) - } - if string(resp1.Header.Peek(HeaderServer)) != expectedServerName { - t.Fatalf("Unexpected server: %q. Expected %q", resp1.Header.Peek(HeaderServer), expectedServerName) - } - if !bytes.Equal(resp1.Body(), []byte(body)) { - t.Fatalf("Unexpected body: %q. Expected %q", resp1.Body(), body) - } -} - -func TestRequestWriteError(t *testing.T) { - t.Parallel() - - // no host - testRequestWriteError(t, "", "/foo/bar", "", "", "") -} - -func testRequestWriteError(t *testing.T, method, requestURI, host, userAgent, body string) { - var req Request - - req.Header.SetMethod(method) - req.Header.SetRequestURI(requestURI) - req.Header.Set(HeaderHost, host) - req.Header.Set(HeaderUserAgent, userAgent) - req.SetBody([]byte(body)) - - w := &bytebufferpool.ByteBuffer{} - bw := bufio.NewWriter(w) - err := req.Write(bw) - if err == nil { - t.Fatalf("Expecting error when writing request=%#v", &req) - } -} - -func testRequestSuccess(t *testing.T, method, requestURI, host, userAgent, body, expectedMethod string) { - var req Request - - req.Header.SetMethod(method) - req.Header.SetRequestURI(requestURI) - req.Header.Set(HeaderHost, host) - req.Header.Set(HeaderUserAgent, userAgent) - req.SetBody([]byte(body)) - - contentType := "foobar" - if method == MethodPost { - req.Header.Set(HeaderContentType, contentType) - } - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - err := req.Write(bw) - if err != nil { - t.Fatalf("Unexpected error when calling Request.Write(): %v", err) - } - if err = bw.Flush(); err != nil { - t.Fatalf("Unexpected error when flushing bufio.Writer: %v", err) - } - - var req1 Request - br := bufio.NewReader(w) - if err = req1.Read(br); err != nil { - t.Fatalf("Unexpected error when calling Request.Read(): %v", err) - } - if string(req1.Header.Method()) != expectedMethod { - t.Fatalf("Unexpected method: %q. Expected %q", req1.Header.Method(), expectedMethod) - } - if len(requestURI) == 0 { - requestURI = "/" - } - if string(req1.Header.RequestURI()) != requestURI { - t.Fatalf("Unexpected RequestURI: %q. Expected %q", req1.Header.RequestURI(), requestURI) - } - if string(req1.Header.Peek(HeaderHost)) != host { - t.Fatalf("Unexpected host: %q. Expected %q", req1.Header.Peek(HeaderHost), host) - } - if string(req1.Header.Peek(HeaderUserAgent)) != userAgent { - t.Fatalf("Unexpected user-agent: %q. Expected %q", req1.Header.Peek(HeaderUserAgent), userAgent) - } - if !bytes.Equal(req1.Body(), []byte(body)) { - t.Fatalf("Unexpected body: %q. Expected %q", req1.Body(), body) - } - - if method == MethodPost && string(req1.Header.Peek(HeaderContentType)) != contentType { - t.Fatalf("Unexpected content-type: %q. Expected %q", req1.Header.Peek(HeaderContentType), contentType) - } -} - -func TestResponseReadSuccess(t *testing.T) { - t.Parallel() - - resp := &Response{} - - // usual response - testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: foo/bar\r\n\r\n0123456789", - 200, 10, "foo/bar", "0123456789", nil) - - // zero response - testResponseReadSuccess(t, resp, "HTTP/1.1 500 OK\r\nContent-Length: 0\r\nContent-Type: foo/bar\r\n\r\n", - 500, 0, "foo/bar", "", nil) - - // response with trailer - testResponseReadSuccess(t, resp, "HTTP/1.1 300 OK\r\nTransfer-Encoding: chunked\r\nContent-Type: bar\r\n\r\n5\r\n56789\r\n0\r\nfoo: bar\r\n\r\n", - 300, -1, "bar", "56789", map[string]string{"Foo": "bar"}) - - // response with trailer disableNormalizing - resp.Header.DisableNormalizing() - testResponseReadSuccess(t, resp, "HTTP/1.1 300 OK\r\nTransfer-Encoding: chunked\r\nContent-Type: bar\r\n\r\n5\r\n56789\r\n0\r\nfoo: bar\r\n\r\n", - 300, -1, "bar", "56789", map[string]string{"foo": "bar"}) - - // no content-length ('identity' transfer-encoding) - testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: foobar\r\n\r\nzxxxx", - 200, 5, "foobar", "zxxxx", nil) - - // explicitly stated 'Transfer-Encoding: identity' - testResponseReadSuccess(t, resp, "HTTP/1.1 234 ss\r\nContent-Type: xxx\r\n\r\nxag", - 234, 3, "xxx", "xag", nil) - - // big 'identity' response - body := string(createFixedBody(100500)) - testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: aa\r\n\r\n"+body, - 200, 100500, "aa", body, nil) - - // chunked response - testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nqwer\r\n2\r\nty\r\n0\r\nFoo2: bar2\r\n\r\n", - 200, -1, "text/html", "qwerty", map[string]string{"Foo2": "bar2"}) - - // chunked response with non-chunked Transfer-Encoding. - testResponseReadSuccess(t, resp, "HTTP/1.1 230 OK\r\nContent-Type: text\r\nTransfer-Encoding: aaabbb\r\n\r\n2\r\ner\r\n2\r\nty\r\n0\r\nFoo3: bar3\r\n\r\n", - 230, -1, "text", "erty", map[string]string{"Foo3": "bar3"}) - - // chunked response with content-length - testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: foo/bar\r\nContent-Length: 123\r\nTransfer-Encoding: chunked\r\n\r\n4\r\ntest\r\n0\r\nFoo4:bar4\r\n\r\n", - 200, -1, "foo/bar", "test", map[string]string{"Foo4": "bar4"}) - - // chunked response with empty body - testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nTransfer-Encoding: chunked\r\n\r\n0\r\nFoo5: bar5\r\n\r\n", - 200, -1, "text/html", "", map[string]string{"Foo5": "bar5"}) - - // chunked response with chunk extension - testResponseReadSuccess(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nTransfer-Encoding: chunked\r\n\r\n3;ext\r\naaa\r\n0\r\nFoo6: bar6\r\n\r\n", - 200, -1, "text/html", "aaa", map[string]string{"Foo6": "bar6"}) - -} - -func TestResponseReadError(t *testing.T) { - t.Parallel() - - resp := &Response{} - - // empty response - testResponseReadError(t, resp, "") - - // invalid header - testResponseReadError(t, resp, "foobar") - - // empty body - testResponseReadError(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: aaa\r\nContent-Length: 1234\r\n\r\n") - - // invalid chunked body - testResponseReadError(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: aaa\r\nContent-Length: 1234\r\n\r\nshort") - - // chunked body without end chunk - testResponseReadError(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: aaa\r\nTransfer-Encoding: chunked\r\n\r\nfoo") - - testResponseReadError(t, resp, "HTTP/1.1 200 OK\r\nContent-Type: aaa\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nfoo") -} - -func testResponseReadError(t *testing.T, resp *Response, response string) { - r := bytes.NewBufferString(response) - rb := bufio.NewReader(r) - err := resp.Read(rb) - if err == nil { - t.Fatalf("Expecting error for response=%q", response) - } - - testResponseReadSuccess(t, resp, "HTTP/1.1 303 Redisred sedfs sdf\r\nContent-Type: aaa\r\nContent-Length: 5\r\n\r\nHELLO", - 303, 5, "aaa", "HELLO", nil) -} - -func testResponseReadSuccess(t *testing.T, resp *Response, response string, expectedStatusCode, expectedContentLength int, - expectedContentType, expectedBody string, expectedTrailer map[string]string) { - - r := bytes.NewBufferString(response) - rb := bufio.NewReader(r) - err := resp.Read(rb) - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - verifyResponseHeader(t, &resp.Header, expectedStatusCode, expectedContentLength, expectedContentType, "") - if !bytes.Equal(resp.Body(), []byte(expectedBody)) { - t.Fatalf("Unexpected body %q. Expected %q", resp.Body(), []byte(expectedBody)) - } - verifyResponseTrailer(t, &resp.Header, expectedTrailer) -} - -func TestReadBodyFixedSize(t *testing.T) { - t.Parallel() - - // zero-size body - testReadBodyFixedSize(t, 0) - - // small-size body - testReadBodyFixedSize(t, 3) - - // medium-size body - testReadBodyFixedSize(t, 1024) - - // large-size body - testReadBodyFixedSize(t, 1024*1024) - - // smaller body after big one - testReadBodyFixedSize(t, 34345) -} - -func TestReadBodyChunked(t *testing.T) { - t.Parallel() - - // zero-size body - testReadBodyChunked(t, 0) - - // small-size body - testReadBodyChunked(t, 5) - - // medium-size body - testReadBodyChunked(t, 43488) - - // big body - testReadBodyChunked(t, 3*1024*1024) - - // smaler body after big one - testReadBodyChunked(t, 12343) -} - -func TestRequestURITLS(t *testing.T) { - t.Parallel() - - uriNoScheme := "//foobar.com/baz/aa?bb=dd&dd#sdf" - requestURI := "http:" + uriNoScheme - requestURITLS := "https:" + uriNoScheme - - var req Request - - req.isTLS = true - req.SetRequestURI(requestURI) - uri := req.PhantomURI().String() - if uri != requestURITLS { - t.Fatalf("unexpected request uri: %q. Expecting %q", uri, requestURITLS) - } - - req.Reset() - req.SetRequestURI(requestURI) - uri = req.PhantomURI().String() - if uri != requestURI { - t.Fatalf("unexpected request uri: %q. Expecting %q", uri, requestURI) - } -} - -func TestRequestURI(t *testing.T) { - t.Parallel() - - host := "foobar.com" - requestURI := "/aaa/bb+b%20d?ccc=ddd&qqq#1334dfds&=d" - expectedPathOriginal := "/aaa/bb+b%20d" - expectedPath := "/aaa/bb+b d" - expectedQueryString := "ccc=ddd&qqq" - expectedHash := "1334dfds&=d" - - var req Request - req.Header.Set(HeaderHost, host) - req.Header.SetRequestURI(requestURI) - - uri := req.PhantomURI() - if string(uri.Host()) != host { - t.Fatalf("Unexpected host %q. Expected %q", uri.Host(), host) - } - if string(uri.PathOriginal()) != expectedPathOriginal { - t.Fatalf("Unexpected source path %q. Expected %q", uri.PathOriginal(), expectedPathOriginal) - } - if string(uri.Path()) != expectedPath { - t.Fatalf("Unexpected path %q. Expected %q", uri.Path(), expectedPath) - } - if string(uri.QueryString()) != expectedQueryString { - t.Fatalf("Unexpected query string %q. Expected %q", uri.QueryString(), expectedQueryString) - } - if string(uri.Hash()) != expectedHash { - t.Fatalf("Unexpected hash %q. Expected %q", uri.Hash(), expectedHash) - } -} - -func TestRequestPostArgsSuccess(t *testing.T) { - t.Parallel() - - var req Request - - testRequestPostArgsSuccess(t, &req, "POST / HTTP/1.1\r\nHost: aaa.com\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 0\r\n\r\n", 0, "foo=", "=") - - testRequestPostArgsSuccess(t, &req, "POST / HTTP/1.1\r\nHost: aaa.com\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: 18\r\n\r\nfoo&b%20r=b+z=&qwe", 3, "foo=", "b r=b z=", "qwe=") -} - -func TestRequestPostArgsError(t *testing.T) { - t.Parallel() - - var req Request - - // non-post - testRequestPostArgsError(t, &req, "GET /aa HTTP/1.1\r\nHost: aaa\r\n\r\n") - - // invalid content-type - testRequestPostArgsError(t, &req, "POST /aa HTTP/1.1\r\nHost: aaa\r\nContent-Type: text/html\r\nContent-Length: 5\r\n\r\nabcde") -} - -func testRequestPostArgsError(t *testing.T, req *Request, s string) { - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - err := req.Read(br) - if err != nil { - t.Fatalf("Unexpected error when reading %q: %v", s, err) - } - ss := req.PostArgs().String() - if len(ss) != 0 { - t.Fatalf("unexpected post args: %q. Expecting empty post args", ss) - } -} - -func testRequestPostArgsSuccess(t *testing.T, req *Request, s string, expectedArgsLen int, expectedArgs ...string) { - r := bytes.NewBufferString(s) - br := bufio.NewReader(r) - err := req.Read(br) - if err != nil { - t.Fatalf("Unexpected error when reading %q: %v", s, err) - } - - args := req.PostArgs() - if args.Len() != expectedArgsLen { - t.Fatalf("Unexpected args len %d. Expected %d for %q", args.Len(), expectedArgsLen, s) - } - for _, x := range expectedArgs { - tmp := strings.SplitN(x, "=", 2) - k := tmp[0] - v := tmp[1] - vv := string(args.Peek(k)) - if vv != v { - t.Fatalf("Unexpected value for key %q: %q. Expected %q for %q", k, vv, v, s) - } - } -} - -func testReadBodyChunked(t *testing.T, bodySize int) { - body := createFixedBody(bodySize) - expectedTrailer := map[string]string{"Foo": "bar"} - chunkedBody := createChunkedBody(body, expectedTrailer, true) - - r := bytes.NewBuffer(chunkedBody) - br := bufio.NewReader(r) - b, err := readBodyChunked(br, 0, nil) - if err != nil { - t.Fatalf("Unexpected error for bodySize=%d: %v. body=%q, chunkedBody=%q", bodySize, err, body, chunkedBody) - } - if !bytes.Equal(b, body) { - t.Fatalf("Unexpected response read for bodySize=%d: %q. Expected %q. chunkedBody=%q", bodySize, b, body, chunkedBody) - } - verifyTrailer(t, br, expectedTrailer, false) -} - -func testReadBodyFixedSize(t *testing.T, bodySize int) { - body := createFixedBody(bodySize) - r := bytes.NewBuffer(body) - br := bufio.NewReader(r) - b, err := readBody(br, bodySize, 0, nil) - if err != nil { - t.Fatalf("Unexpected error in ReadResponseBody(%d): %v", bodySize, err) - } - if !bytes.Equal(b, body) { - t.Fatalf("Unexpected response read for bodySize=%d: %q. Expected %q", bodySize, b, body) - } - verifyTrailer(t, br, nil, false) -} - -func createFixedBody(bodySize int) []byte { - var b []byte - for i := 0; i < bodySize; i++ { - b = append(b, byte(i%10)+'0') - } - return b -} - -func createChunkedBody(body []byte, trailer map[string]string, withEnd bool) []byte { - var b []byte - chunkSize := 1 - for len(body) > 0 { - if chunkSize > len(body) { - chunkSize = len(body) - } - b = append(b, []byte(fmt.Sprintf("%x\r\n", chunkSize))...) - b = append(b, body[:chunkSize]...) - b = append(b, []byte("\r\n")...) - body = body[chunkSize:] - chunkSize++ - } - if withEnd { - b = append(b, "0\r\n"...) - for k, v := range trailer { - b = append(b, k...) - b = append(b, ": "...) - b = append(b, v...) - b = append(b, "\r\n"...) - } - b = append(b, "\r\n"...) - } - return b -} - -func TestWriteMultipartForm(t *testing.T) { - t.Parallel() - - var w bytes.Buffer - s := strings.Replace(`--foo -Content-Disposition: form-data; name="key" - -value ---foo -Content-Disposition: form-data; name="file"; filename="test.json" -Content-Type: application/json - -{"foo": "bar"} ---foo-- -`, "\n", "\r\n", -1) - mr := multipart.NewReader(strings.NewReader(s), "foo") - form, err := mr.ReadForm(1024) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if err := WriteMultipartForm(&w, form, "foo"); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if w.String() != s { - t.Fatalf("unexpected output %q", w.Bytes()) - } -} - -func TestResponseRawBodySet(t *testing.T) { - t.Parallel() - - var resp Response - - expectedS := "test" - body := []byte(expectedS) - resp.SetBody(body) - - testBodyWriteTo(t, &resp, expectedS, true) -} - -func TestRequestRawBodySet(t *testing.T) { - t.Parallel() - - var r Request - - expectedS := "test" - body := []byte(expectedS) - r.SetBody(body) - - testBodyWriteTo(t, &r, expectedS, true) -} - -func TestResponseRawBodyReset(t *testing.T) { - t.Parallel() - - var resp Response - - body := []byte("test") - resp.SetBody(body) - resp.ResetBody() - - testBodyWriteTo(t, &resp, "", true) -} - -func TestRequestRawBodyReset(t *testing.T) { - t.Parallel() - - var r Request - - body := []byte("test") - r.SetBody(body) - r.ResetBody() - - testBodyWriteTo(t, &r, "", true) -} - -func TestResponseRawBodyCopyTo(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var resp Response - - expectedS := "test" - body := []byte(expectedS) - resp.SetBody(body) - - testResponseCopyTo(t, &resp) -} - -func TestRequestRawBodyCopyTo(t *testing.T) { - t.Parallel() - - var a Request - - body := []byte("test") - a.SetBody(body) - - var b Request - - a.CopyTo(&b) - - testBodyWriteTo(t, &a, "test", true) - testBodyWriteTo(t, &b, "test", true) -} - -type testReader struct { - read chan (int) - cb chan (struct{}) - onClose func() error -} - -func (r *testReader) Read(b []byte) (int, error) { - read := <-r.read - - if read == -1 { - return 0, io.EOF - } - - r.cb <- struct{}{} - - for i := 0; i < read; i++ { - b[i] = 'x' - } - - return read, nil -} - -func (r *testReader) Close() error { - if r.onClose != nil { - return r.onClose() - } - return nil -} - -func TestResponseImmediateHeaderFlushRegressionFixedLength(t *testing.T) { - t.Parallel() - - var r Response - - expectedS := "aaabbbccc" - buf := bytes.NewBufferString(expectedS) - r.SetBodyStream(buf, len(expectedS)) - r.ImmediateHeaderFlush = true - - testBodyWriteTo(t, &r, expectedS, false) -} - -func TestResponseImmediateHeaderFlushRegressionChunked(t *testing.T) { - t.Parallel() - - var r Response - - expectedS := "aaabbbccc" - buf := bytes.NewBufferString(expectedS) - r.SetBodyStream(buf, -1) - r.ImmediateHeaderFlush = true - - testBodyWriteTo(t, &r, expectedS, false) -} - -func TestResponseImmediateHeaderFlushFixedLength(t *testing.T) { - t.Parallel() - - var r Response - - r.ImmediateHeaderFlush = true - - ch := make(chan int) - cb := make(chan struct{}) - - buf := &testReader{read: ch, cb: cb} - - r.SetBodyStream(buf, 3) - - b := []byte{} - w := bytes.NewBuffer(b) - bb := bufio.NewWriter(w) - - bw := &r - - waitForIt := make(chan struct{}) - - go func() { - if err := bw.Write(bb); err != nil { - t.Errorf("unexpected error: %v", err) - } - waitForIt <- struct{}{} - }() - - ch <- 3 - - if !strings.Contains(w.String(), "Content-Length: 3") { - t.Fatalf("Expected headers to be flushed") - } - - if strings.Contains(w.String(), "xxx") { - t.Fatalf("Did not expext body to be written yet") - } - - <-cb - ch <- -1 - - <-waitForIt -} - -func TestResponseImmediateHeaderFlushFixedLengthSkipBody(t *testing.T) { - t.Parallel() - - var r Response - - r.ImmediateHeaderFlush = true - r.SkipBody = true - - ch := make(chan int) - cb := make(chan struct{}) - - buf := &testReader{read: ch, cb: cb} - - r.SetBodyStream(buf, 0) - - b := []byte{} - w := bytes.NewBuffer(b) - bb := bufio.NewWriter(w) - - var headersOnClose string - buf.onClose = func() error { - headersOnClose = w.String() - return nil - } - - bw := &r - - if err := bw.Write(bb); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if !strings.Contains(headersOnClose, "Content-Length: 0") { - t.Fatalf("Expected headers to be eagerly flushed") - } -} - -func TestResponseImmediateHeaderFlushChunked(t *testing.T) { - t.Parallel() - - var r Response - - r.ImmediateHeaderFlush = true - - ch := make(chan int) - cb := make(chan struct{}) - - buf := &testReader{read: ch, cb: cb} - - r.SetBodyStream(buf, -1) - - b := []byte{} - w := bytes.NewBuffer(b) - bb := bufio.NewWriter(w) - - bw := &r - - waitForIt := make(chan struct{}) - - go func() { - if err := bw.Write(bb); err != nil { - t.Errorf("unexpected error: %v", err) - } - - waitForIt <- struct{}{} - }() - - ch <- 3 - - if !strings.Contains(w.String(), "Transfer-Encoding: chunked") { - t.Fatalf("Expected headers to be flushed") - } - - if strings.Contains(w.String(), "xxx") { - t.Fatalf("Did not expext body to be written yet") - } - - <-cb - ch <- -1 - - <-waitForIt -} - -func TestResponseImmediateHeaderFlushChunkedNoBody(t *testing.T) { - t.Parallel() - - var r Response - - r.ImmediateHeaderFlush = true - r.SkipBody = true - - ch := make(chan int) - cb := make(chan struct{}) - - buf := &testReader{read: ch, cb: cb} - - r.SetBodyStream(buf, -1) - - b := []byte{} - w := bytes.NewBuffer(b) - bb := bufio.NewWriter(w) - - var headersOnClose string - buf.onClose = func() error { - headersOnClose = w.String() - return nil - } - - bw := &r - - if err := bw.Write(bb); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if !strings.Contains(headersOnClose, "Transfer-Encoding: chunked") { - t.Fatalf("Expected headers to be eagerly flushed") - } -} - -type ErroneousBodyStream struct { - errOnRead bool - errOnClose bool -} - -func (ebs *ErroneousBodyStream) Read(p []byte) (n int, err error) { - if ebs.errOnRead { - panic("reading erroneous body stream") - } - return 0, io.EOF -} - -func (ebs *ErroneousBodyStream) Close() error { - if ebs.errOnClose { - panic("closing erroneous body stream") - } - return nil -} - -func TestResponseBodyStreamErrorOnPanicDuringRead(t *testing.T) { - t.Parallel() - var resp Response - var w bytes.Buffer - bw := bufio.NewWriter(&w) - - ebs := &ErroneousBodyStream{errOnRead: true, errOnClose: false} - resp.SetBodyStream(ebs, 42) - err := resp.Write(bw) - if err == nil { - t.Fatalf("expected error when writing response.") - } - e, ok := err.(*ErrBodyStreamWritePanic) - if !ok { - t.Fatalf("expected error struct to be *ErrBodyStreamWritePanic, got: %+v.", e) - } - if e.Error() != "panic while writing body stream: reading erroneous body stream" { - t.Fatalf("unexpected error value, got: %+v.", e.Error()) - } -} - -func TestResponseBodyStreamErrorOnPanicDuringClose(t *testing.T) { - t.Parallel() - var resp Response - var w bytes.Buffer - bw := bufio.NewWriter(&w) - - ebs := &ErroneousBodyStream{errOnRead: false, errOnClose: true} - resp.SetBodyStream(ebs, 42) - err := resp.Write(bw) - if err == nil { - t.Fatalf("expected error when writing response.") - } - e, ok := err.(*ErrBodyStreamWritePanic) - if !ok { - t.Fatalf("expected error struct to be *ErrBodyStreamWritePanic, got: %+v.", e) - } - if e.Error() != "panic while writing body stream: closing erroneous body stream" { - t.Fatalf("unexpected error value, got: %+v.", e.Error()) - } -} diff --git a/lib/fasthttp/lbclient_example_test.go b/lib/fasthttp/lbclient_example_test.go deleted file mode 100644 index 03c72a993..000000000 --- a/lib/fasthttp/lbclient_example_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package fasthttp_test - -import ( - "fmt" - "log" - - "infini.sh/framework/lib/fasthttp" -) - -func ExampleLBClient() { - // Requests will be spread among these servers. - servers := []string{ - "google.com:80", - "foobar.com:8080", - "127.0.0.1:123", - } - - // Prepare clients for each server - var lbc fasthttp.LBClient - for _, addr := range servers { - c := &fasthttp.HostClient{ - Addr: addr, - } - lbc.Clients = append(lbc.Clients, c) - } - - // Send requests to load-balanced servers - var req fasthttp.Request - var resp fasthttp.Response - for i := 0; i < 10; i++ { - url := fmt.Sprintf("http://abcedfg/foo/bar/%d", i) - req.SetRequestURI(url) - if err := lbc.Do(&req, &resp); err != nil { - log.Fatalf("Error when sending request: %v", err) - } - if resp.StatusCode() != fasthttp.StatusOK { - log.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), fasthttp.StatusOK) - } - - useResponseBody(resp.Body()) - } -} diff --git a/lib/fasthttp/peripconn_test.go b/lib/fasthttp/peripconn_test.go deleted file mode 100644 index e2137c0f8..000000000 --- a/lib/fasthttp/peripconn_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package fasthttp - -import ( - "testing" -) - -func TestIPxUint32(t *testing.T) { - t.Parallel() - - testIPxUint32(t, 0) - testIPxUint32(t, 10) - testIPxUint32(t, 0x12892392) -} - -func testIPxUint32(t *testing.T, n uint32) { - ip := uint322ip(n) - nn := ip2uint32(ip) - if n != nn { - t.Fatalf("Unexpected value=%d for ip=%q. Expected %d", nn, ip, n) - } -} - -func TestPerIPConnCounter(t *testing.T) { - t.Parallel() - - var cc perIPConnCounter - - expectPanic(t, func() { cc.Unregister(123) }) - - for i := 1; i < 100; i++ { - if n := cc.Register(123); n != i { - t.Fatalf("Unexpected counter value=%d. Expected %d", n, i) - } - } - - n := cc.Register(456) - if n != 1 { - t.Fatalf("Unexpected counter value=%d. Expected 1", n) - } - - for i := 1; i < 100; i++ { - cc.Unregister(123) - } - cc.Unregister(456) - - expectPanic(t, func() { cc.Unregister(123) }) - expectPanic(t, func() { cc.Unregister(456) }) - - n = cc.Register(123) - if n != 1 { - t.Fatalf("Unexpected counter value=%d. Expected 1", n) - } - cc.Unregister(123) -} - -func expectPanic(t *testing.T, f func()) { - defer func() { - if r := recover(); r == nil { - t.Fatalf("Expecting panic") - } - }() - f() -} diff --git a/lib/fasthttp/prefork/prefork_test.go b/lib/fasthttp/prefork/prefork_test.go deleted file mode 100644 index 39489f655..000000000 --- a/lib/fasthttp/prefork/prefork_test.go +++ /dev/null @@ -1,228 +0,0 @@ -package prefork - -import ( - "fmt" - "math/rand" - "net" - "os" - "reflect" - "runtime" - "testing" - - "infini.sh/framework/lib/fasthttp" -) - -func setUp() { - os.Args = append(os.Args, preforkChildFlag) -} - -func tearDown() { - os.Args = os.Args[:len(os.Args)-1] -} - -func getAddr() string { - return fmt.Sprintf("0.0.0.0:%d", rand.Intn(9000-3000)+3000) -} - -func Test_IsChild(t *testing.T) { - // This test can't run parallel as it modifies os.Args. - - v := IsChild() - if v { - t.Errorf("IsChild() == %v, want %v", v, false) - } - - setUp() - defer tearDown() - - v = IsChild() - if !v { - t.Errorf("IsChild() == %v, want %v", v, true) - } -} - -func Test_New(t *testing.T) { - t.Parallel() - - s := &fasthttp.Server{} - p := New(s) - - if p.Network != defaultNetwork { - t.Errorf("Prefork.Netork == %q, want %q", p.Network, defaultNetwork) - } - - if reflect.ValueOf(p.ServeFunc).Pointer() != reflect.ValueOf(s.Serve).Pointer() { - t.Errorf("Prefork.ServeFunc == %p, want %p", p.ServeFunc, s.Serve) - } - - if reflect.ValueOf(p.ServeTLSFunc).Pointer() != reflect.ValueOf(s.ServeTLS).Pointer() { - t.Errorf("Prefork.ServeTLSFunc == %p, want %p", p.ServeTLSFunc, s.ServeTLS) - } - - if reflect.ValueOf(p.ServeTLSEmbedFunc).Pointer() != reflect.ValueOf(s.ServeTLSEmbed).Pointer() { - t.Errorf("Prefork.ServeTLSFunc == %p, want %p", p.ServeTLSEmbedFunc, s.ServeTLSEmbed) - } -} - -func Test_listen(t *testing.T) { - t.Parallel() - - p := &Prefork{ - Reuseport: true, - } - addr := getAddr() - - ln, err := p.listen(addr) - - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - ln.Close() - - lnAddr := ln.Addr().String() - if lnAddr != addr { - t.Errorf("Prefork.Addr == %q, want %q", lnAddr, addr) - } - - if p.Network != defaultNetwork { - t.Errorf("Prefork.Network == %q, want %q", p.Network, defaultNetwork) - } - - procs := runtime.GOMAXPROCS(0) - if procs != 1 { - t.Errorf("GOMAXPROCS == %d, want %d", procs, 1) - } -} - -func Test_setTCPListenerFiles(t *testing.T) { - t.Parallel() - - if runtime.GOOS == "windows" { - t.SkipNow() - } - - p := &Prefork{} - addr := getAddr() - - err := p.setTCPListenerFiles(addr) - - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - - if p.ln == nil { - t.Fatal("Prefork.ln is nil") - } - - p.ln.Close() - - lnAddr := p.ln.Addr().String() - if lnAddr != addr { - t.Errorf("Prefork.Addr == %q, want %q", lnAddr, addr) - } - - if p.Network != defaultNetwork { - t.Errorf("Prefork.Network == %q, want %q", p.Network, defaultNetwork) - } - - if len(p.files) != 1 { - t.Errorf("Prefork.files == %d, want %d", len(p.files), 1) - } -} - -func Test_ListenAndServe(t *testing.T) { - // This test can't run parallel as it modifies os.Args. - - setUp() - defer tearDown() - - s := &fasthttp.Server{} - p := New(s) - p.Reuseport = true - p.ServeFunc = func(ln net.Listener) error { - return nil - } - - addr := getAddr() - - err := p.ListenAndServe(addr) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - p.ln.Close() - - lnAddr := p.ln.Addr().String() - if lnAddr != addr { - t.Errorf("Prefork.Addr == %q, want %q", lnAddr, addr) - } - - if p.ln == nil { - t.Error("Prefork.ln is nil") - } -} - -func Test_ListenAndServeTLS(t *testing.T) { - // This test can't run parallel as it modifies os.Args. - - setUp() - defer tearDown() - - s := &fasthttp.Server{} - p := New(s) - p.Reuseport = true - p.ServeTLSFunc = func(ln net.Listener, certFile, keyFile string) error { - return nil - } - - addr := getAddr() - - err := p.ListenAndServeTLS(addr, "./key", "./cert") - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - p.ln.Close() - - lnAddr := p.ln.Addr().String() - if lnAddr != addr { - t.Errorf("Prefork.Addr == %q, want %q", lnAddr, addr) - } - - if p.ln == nil { - t.Error("Prefork.ln is nil") - } -} - -func Test_ListenAndServeTLSEmbed(t *testing.T) { - // This test can't run parallel as it modifies os.Args. - - setUp() - defer tearDown() - - s := &fasthttp.Server{} - p := New(s) - p.Reuseport = true - p.ServeTLSEmbedFunc = func(ln net.Listener, certData, keyData []byte) error { - return nil - } - - addr := getAddr() - - err := p.ListenAndServeTLSEmbed(addr, []byte("key"), []byte("cert")) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - p.ln.Close() - - lnAddr := p.ln.Addr().String() - if lnAddr != addr { - t.Errorf("Prefork.Addr == %q, want %q", lnAddr, addr) - } - - if p.ln == nil { - t.Error("Prefork.ln is nil") - } -} diff --git a/lib/fasthttp/request_context_test.go b/lib/fasthttp/request_context_test.go deleted file mode 100644 index e9d375443..000000000 --- a/lib/fasthttp/request_context_test.go +++ /dev/null @@ -1,13 +0,0 @@ -/* Copyright © INFINI LTD. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package fasthttp - -import "testing" - -func TestCtxEncode(t *testing.T) { - ctx := RequestCtx{} - ctx.Request = Request{} - ctx.Request.SetRequestURI("/favicon.ico") -} diff --git a/lib/fasthttp/requestctx_setbodystreamwriter_example_test.go b/lib/fasthttp/requestctx_setbodystreamwriter_example_test.go deleted file mode 100644 index dc3013ec2..000000000 --- a/lib/fasthttp/requestctx_setbodystreamwriter_example_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package fasthttp_test - -import ( - "bufio" - "fmt" - "log" - "time" - - "infini.sh/framework/lib/fasthttp" -) - -func ExampleRequestCtx_SetBodyStreamWriter() { - // Start fasthttp server for streaming responses. - if err := fasthttp.ListenAndServe(":8080", responseStreamHandler); err != nil { - log.Fatalf("unexpected error in server: %v", err) - } -} - -func responseStreamHandler(ctx *fasthttp.RequestCtx) { - // Send the response in chunks and wait for a second between each chunk. - ctx.SetBodyStreamWriter(func(w *bufio.Writer) { - for i := 0; i < 10; i++ { - fmt.Fprintf(w, "this is a message number %d", i) - - // Do not forget flushing streamed data to the client. - if err := w.Flush(); err != nil { - return - } - time.Sleep(time.Second) - } - }) -} diff --git a/lib/fasthttp/reuseport/reuseport_example_test.go b/lib/fasthttp/reuseport/reuseport_example_test.go deleted file mode 100644 index 6581f44dc..000000000 --- a/lib/fasthttp/reuseport/reuseport_example_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package reuseport_test - -import ( - "fmt" - "log" - - "infini.sh/framework/lib/fasthttp" - "infini.sh/framework/lib/fasthttp/reuseport" -) - -func ExampleListen() { - ln, err := reuseport.Listen("tcp4", "localhost:12345") - if err != nil { - log.Fatalf("error in reuseport listener: %v", err) - } - - if err = fasthttp.Serve(ln, requestHandler); err != nil { - log.Fatalf("error in fasthttp Server: %v", err) - } -} - -func requestHandler(ctx *fasthttp.RequestCtx) { - fmt.Fprintf(ctx, "Hello, world!") -} diff --git a/lib/fasthttp/reuseport/reuseport_test.go b/lib/fasthttp/reuseport/reuseport_test.go deleted file mode 100644 index 8ebf3d841..000000000 --- a/lib/fasthttp/reuseport/reuseport_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package reuseport - -import ( - "net" - "testing" -) - -func TestTCP4(t *testing.T) { - t.Parallel() - - testNewListener(t, "tcp4", "localhost:10081") -} - -func TestTCP6(t *testing.T) { - t.Parallel() - - // Run this test only if tcp6 interface exists. - if hasLocalIPv6(t) { - testNewListener(t, "tcp6", "[::1]:10082") - } -} - -func hasLocalIPv6(t *testing.T) bool { - addrs, err := net.InterfaceAddrs() - if err != nil { - t.Fatalf("cannot obtain local interfaces: %v", err) - } - for _, a := range addrs { - if a.String() == "::1/128" { - return true - } - } - return false -} - -func testNewListener(t *testing.T, network, addr string) { - ln1, err := Listen(network, addr) - if err != nil { - t.Fatalf("cannot create listener %v", err) - } - - ln2, err := Listen(network, addr) - if err != nil { - t.Fatalf("cannot create listener %v", err) - } - - _ = ln1.Close() - _ = ln2.Close() -} diff --git a/lib/fasthttp/server_example_test.go b/lib/fasthttp/server_example_test.go deleted file mode 100644 index 3907e8917..000000000 --- a/lib/fasthttp/server_example_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package fasthttp_test - -import ( - "fmt" - "log" - "math/rand" - "net" - "time" - - "infini.sh/framework/lib/fasthttp" -) - -func ExampleListenAndServe() { - // The server will listen for incoming requests on this address. - listenAddr := "127.0.0.1:80" - - // This function will be called by the server for each incoming request. - // - // RequestCtx provides a lot of functionality related to http request - // processing. See RequestCtx docs for details. - requestHandler := func(ctx *fasthttp.RequestCtx) { - fmt.Fprintf(ctx, "Hello, world! Requested path is %q", ctx.Path()) - } - - // Start the server with default settings. - // Create Server instance for adjusting server settings. - // - // ListenAndServe returns only on error, so usually it blocks forever. - if err := fasthttp.ListenAndServe(listenAddr, requestHandler); err != nil { - log.Fatalf("error in ListenAndServe: %v", err) - } -} - -func ExampleServe() { - // Create network listener for accepting incoming requests. - // - // Note that you are not limited by TCP listener - arbitrary - // net.Listener may be used by the server. - // For example, unix socket listener or TLS listener. - ln, err := net.Listen("tcp4", "127.0.0.1:8080") - if err != nil { - log.Fatalf("error in net.Listen: %v", err) - } - - // This function will be called by the server for each incoming request. - // - // RequestCtx provides a lot of functionality related to http request - // processing. See RequestCtx docs for details. - requestHandler := func(ctx *fasthttp.RequestCtx) { - fmt.Fprintf(ctx, "Hello, world! Requested path is %q", ctx.Path()) - } - - // Start the server with default settings. - // Create Server instance for adjusting server settings. - // - // Serve returns on ln.Close() or error, so usually it blocks forever. - if err := fasthttp.Serve(ln, requestHandler); err != nil { - log.Fatalf("error in Serve: %v", err) - } -} - -func ExampleServer() { - // This function will be called by the server for each incoming request. - // - // RequestCtx provides a lot of functionality related to http request - // processing. See RequestCtx docs for details. - requestHandler := func(ctx *fasthttp.RequestCtx) { - fmt.Fprintf(ctx, "Hello, world! Requested path is %q", ctx.Path()) - } - - // Create custom server. - s := &fasthttp.Server{ - Handler: requestHandler, - - // Every response will contain 'Server: My super server' header. - Name: "My super server", - - // Other Server settings may be set here. - } - - // Start the server listening for incoming requests on the given address. - // - // ListenAndServe returns only on error, so usually it blocks forever. - if err := s.ListenAndServe("127.0.0.1:80"); err != nil { - log.Fatalf("error in ListenAndServe: %v", err) - } -} - -func ExampleRequestCtx_Hijack() { - // hijackHandler is called on hijacked connection. - hijackHandler := func(c net.Conn) { - fmt.Fprintf(c, "This message is sent over a hijacked connection to the client %s\n", c.RemoteAddr()) - fmt.Fprintf(c, "Send me something and I'll echo it to you\n") - var buf [1]byte - for { - if _, err := c.Read(buf[:]); err != nil { - log.Printf("error when reading from hijacked connection: %v", err) - return - } - fmt.Fprintf(c, "You sent me %q. Waiting for new data\n", buf[:]) - } - } - - // requestHandler is called for each incoming request. - requestHandler := func(ctx *fasthttp.RequestCtx) { - path := ctx.Path() - switch { - case string(path) == "/hijack": - // Note that the connection is hijacked only after - // returning from requestHandler and sending http response. - ctx.Hijack(hijackHandler) - - // The connection will be hijacked after sending this response. - fmt.Fprintf(ctx, "Hijacked the connection!") - case string(path) == "/": - fmt.Fprintf(ctx, "Root directory requested") - default: - fmt.Fprintf(ctx, "Requested path is %q", path) - } - } - - if err := fasthttp.ListenAndServe(":80", requestHandler); err != nil { - log.Fatalf("error in ListenAndServe: %v", err) - } -} - -func ExampleRequestCtx_TimeoutError() { - requestHandler := func(ctx *fasthttp.RequestCtx) { - // Emulate long-running task, which touches ctx. - doneCh := make(chan struct{}) - go func() { - workDuration := time.Millisecond * time.Duration(rand.Intn(2000)) - time.Sleep(workDuration) - - fmt.Fprintf(ctx, "ctx has been accessed by long-running task\n") - fmt.Fprintf(ctx, "The reuqestHandler may be finished by this time.\n") - - close(doneCh) - }() - - select { - case <-doneCh: - fmt.Fprintf(ctx, "The task has been finished in less than a second") - case <-time.After(time.Second): - // Since the long-running task is still running and may access ctx, - // we must call TimeoutError before returning from requestHandler. - // - // Otherwise the program will suffer from data races. - ctx.TimeoutError("Timeout!") - } - } - - if err := fasthttp.ListenAndServe(":80", requestHandler); err != nil { - log.Fatalf("error in ListenAndServe: %v", err) - } -} diff --git a/lib/fasthttp/server_test.go b/lib/fasthttp/server_test.go deleted file mode 100644 index fdf6ed2f1..000000000 --- a/lib/fasthttp/server_test.go +++ /dev/null @@ -1,4182 +0,0 @@ -// go:build !windows || !race - -package fasthttp - -import ( - "bufio" - "bytes" - "context" - "crypto/tls" - "errors" - "fmt" - "io" - "io/ioutil" - "mime/multipart" - "net" - "os" - "reflect" - "regexp" - "runtime" - "strings" - "sync" - "testing" - "time" - - "infini.sh/framework/lib/fasthttp/fasthttputil" -) - -// Make sure RequestCtx implements context.Context -var _ context.Context = &RequestCtx{} - -type closerWithRequestCtx struct { - ctx *RequestCtx - closeFunc func(ctx *RequestCtx) error -} - -func (c *closerWithRequestCtx) Close() error { - return c.closeFunc(c.ctx) -} - -func TestServerCRNLAfterPost_Pipeline(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - Logger: &testLogger{}, - } - - ln := fasthttputil.NewInmemoryListener() - defer ln.Close() - - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - }() - - c, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer c.Close() - if _, err = c.Write([]byte("POST / HTTP/1.1\r\nHost: golang.org\r\nContent-Length: 3\r\n\r\nABC" + - "\r\n\r\n" + // <-- this stuff is bogus, but we'll ignore it - "GET / HTTP/1.1\r\nHost: golang.org\r\n\r\n")); err != nil { - t.Fatal(err) - } - - br := bufio.NewReader(c) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } -} - -func TestServerCRNLAfterPost(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - Logger: &testLogger{}, - ReadTimeout: time.Millisecond * 100, - } - - ln := fasthttputil.NewInmemoryListener() - defer ln.Close() - - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - }() - - c, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - defer c.Close() - if _, err = c.Write([]byte("POST / HTTP/1.1\r\nHost: golang.org\r\nContent-Length: 3\r\n\r\nABC" + - "\r\n\r\n", // <-- this stuff is bogus, but we'll ignore it - )); err != nil { - t.Fatal(err) - } - - br := bufio.NewReader(c) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - if err := resp.Read(br); err == nil { - t.Fatal("expected error") // We didn't send a request so we should get an error here. - } -} - -func TestServerPipelineFlush(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - } - ln := fasthttputil.NewInmemoryListener() - - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - }() - - c, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if _, err = c.Write([]byte("GET /foo1 HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Fatal(err) - } - - // Write a partial request. - if _, err = c.Write([]byte("GET /foo1 HTTP/1.1\r\nHost: ")); err != nil { - t.Fatal(err) - } - go func() { - // Wait for 200ms to finish the request - time.Sleep(time.Millisecond * 200) - - if _, err = c.Write([]byte("google.com\r\n\r\n")); err != nil { - t.Error(err) - } - }() - - start := time.Now() - br := bufio.NewReader(c) - var resp Response - - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - - // Since the second request takes 200ms to finish we expect the first one to be flushed earlier. - d := time.Since(start) - if d >= time.Millisecond*200 { - t.Fatalf("had to wait for %v", d) - } - - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } -} - -func TestServerInvalidHeader(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - if ctx.Request.Header.Peek("Foo") != nil || ctx.Request.Header.Peek("Foo ") != nil { - t.Error("expected Foo header") - } - }, - Logger: &testLogger{}, - } - - ln := fasthttputil.NewInmemoryListener() - - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - }() - - c, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if _, err = c.Write([]byte("POST /foo HTTP/1.1\r\nHost: gle.com\r\nFoo : bar\r\nContent-Length: 5\r\n\r\n12345")); err != nil { - t.Fatal(err) - } - - br := bufio.NewReader(c) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusBadRequest { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusBadRequest) - } - - c, err = ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if _, err = c.Write([]byte("GET /foo HTTP/1.1\r\nHost: gle.com\r\nFoo : bar\r\n\r\n")); err != nil { - t.Fatal(err) - } - - br = bufio.NewReader(c) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if resp.StatusCode() != StatusBadRequest { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusBadRequest) - } - - if err := c.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestServerConnState(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - states := make([]string, 0) - s := &Server{ - Handler: func(ctx *RequestCtx) {}, - ConnState: func(_ net.Conn, state ConnState) { - states = append(states, state.String()) - }, - } - - ln := fasthttputil.NewInmemoryListener() - - serverCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverCh) - }() - - clientCh := make(chan struct{}) - go func() { - c, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(c) - // Send 2 requests on the same connection. - for i := 0; i < 2; i++ { - if _, err = c.Write([]byte("GET / HTTP/1.1\r\nHost: aa\r\n\r\n")); err != nil { - t.Errorf("unexpected error: %v", err) - } - var resp Response - if err := resp.Read(br); err != nil { - t.Errorf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - } - if err := c.Close(); err != nil { - t.Errorf("unexpected error: %v", err) - } - // Give the server a little bit of time to transition the connection to the close state. - time.Sleep(time.Millisecond * 100) - close(clientCh) - }() - - select { - case <-clientCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - // 2 requests so we go to active and idle twice. - expected := []string{"new", "active", "idle", "active", "idle", "closed"} - - if !reflect.DeepEqual(expected, states) { - t.Fatalf("wrong state, expected %q, got %q", expected, states) - } -} - -func TestSaveMultipartFile(t *testing.T) { - t.Parallel() - - filea := "This is a test file." - fileb := strings.Repeat("test", 64) - - mr := multipart.NewReader(strings.NewReader(""+ - "--foo\r\n"+ - "Content-Disposition: form-data; name=\"filea\"; filename=\"filea.txt\"\r\n"+ - "Content-Type: text/plain\r\n"+ - "\r\n"+ - filea+"\r\n"+ - "--foo\r\n"+ - "Content-Disposition: form-data; name=\"fileb\"; filename=\"fileb.txt\"\r\n"+ - "Content-Type: text/plain\r\n"+ - "\r\n"+ - fileb+"\r\n"+ - "--foo--\r\n", - ), "foo") - - f, err := mr.ReadForm(64) - if err != nil { - t.Fatal(err) - } - - if err := SaveMultipartFile(f.File["filea"][0], "filea.txt"); err != nil { - t.Fatal(err) - } - defer os.Remove("filea.txt") - - if c, err := ioutil.ReadFile("filea.txt"); err != nil { - t.Fatal(err) - } else if string(c) != filea { - t.Fatalf("filea changed expected %q got %q", filea, c) - } - - // Make sure fileb was saved to a file. - if ff, err := f.File["fileb"][0].Open(); err != nil { - t.Fatal("expected FileHeader.Open to work") - } else if _, ok := ff.(*os.File); !ok { - t.Fatal("expected fileb to be an os.File") - } else { - ff.Close() - } - - if err := SaveMultipartFile(f.File["fileb"][0], "fileb.txt"); err != nil { - t.Fatal(err) - } - defer os.Remove("fileb.txt") - - if c, err := ioutil.ReadFile("fileb.txt"); err != nil { - t.Fatal(err) - } else if string(c) != fileb { - t.Fatalf("fileb changed expected %q got %q", fileb, c) - } -} - -func TestServerName(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - }, - } - - getReponse := func() []byte { - rw := &readWriter{} - rw.r.WriteString("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - resp, err := ioutil.ReadAll(&rw.w) - if err != nil { - t.Fatalf("Unexpected error from ReadAll: %v", err) - } - - return resp - } - - resp := getReponse() - if !bytes.Contains(resp, []byte("\r\nServer: "+string(defaultServerName)+"\r\n")) { - t.Fatalf("Unexpected response %q expected Server: "+string(defaultServerName), resp) - } - - // We can't just overwrite s.Name as fasthttp caches the name in an atomic.Value - s = &Server{ - Handler: func(ctx *RequestCtx) { - }, - Name: "foobar", - } - - resp = getReponse() - if !bytes.Contains(resp, []byte("\r\nServer: foobar\r\n")) { - t.Fatalf("Unexpected response %q expected Server: foobar", resp) - } - - s = &Server{ - Handler: func(ctx *RequestCtx) { - }, - NoDefaultServerHeader: true, - NoDefaultContentType: true, - NoDefaultDate: true, - } - - resp = getReponse() - if bytes.Contains(resp, []byte("\r\nServer: ")) { - t.Fatalf("Unexpected response %q expected no Server header", resp) - } - - if bytes.Contains(resp, []byte("\r\nContent-Type: ")) { - t.Fatalf("Unexpected response %q expected no Content-Type header", resp) - } - - if bytes.Contains(resp, []byte("\r\nDate: ")) { - t.Fatalf("Unexpected response %q expected no Date header", resp) - } -} - -func TestRequestCtxString(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - - s := ctx.String() - expectedS := "#0000000000000000 - 0.0.0.0:0<->0.0.0.0:0 - GET http:///" - if s != expectedS { - t.Fatalf("unexpected ctx.String: %q. Expecting %q", s, expectedS) - } - - ctx.Request.SetRequestURI("https://foobar.com/aaa?bb=c") - s = ctx.String() - expectedS = "#0000000000000000 - 0.0.0.0:0<->0.0.0.0:0 - GET https://foobar.com/aaa?bb=c" - if s != expectedS { - t.Fatalf("unexpected ctx.String: %q. Expecting %q", s, expectedS) - } -} - -func TestServerErrSmallBuffer(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.WriteString("shouldn't be never called") //nolint:errcheck - }, - ReadBufferSize: 20, - } - - rw := &readWriter{} - rw.r.WriteString("GET / HTTP/1.1\r\nHost: aabb.com\r\nVERY-long-Header: sdfdfsd dsf dsaf dsf df fsd\r\n\r\n") - - ch := make(chan error) - go func() { - ch <- s.ServeConn(rw) - }() - - var serverErr error - select { - case serverErr = <-ch: - case <-time.After(200 * time.Millisecond): - t.Fatal("timeout") - } - - if serverErr == nil { - t.Fatal("expected error") - } - - br := bufio.NewReader(&rw.w) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - statusCode := resp.StatusCode() - if statusCode != StatusRequestHeaderFieldsTooLarge { - t.Fatalf("unexpected status code: %d. Expecting %d", statusCode, StatusRequestHeaderFieldsTooLarge) - } - if !resp.ConnectionClose() { - t.Fatal("missing 'Connection: close' response header") - } - - expectedErr := errSmallBuffer.Error() - if !strings.Contains(serverErr.Error(), expectedErr) { - t.Fatalf("unexpected log output: %v. Expecting %q", serverErr, expectedErr) - } -} - -func TestRequestCtxIsTLS(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - - // tls.Conn - ctx.c = &tls.Conn{} - if !ctx.IsTLS() { - t.Fatal("IsTLS must return true") - } - - // non-tls.Conn - ctx.c = &readWriter{} - if ctx.IsTLS() { - t.Fatal("IsTLS must return false") - } - - // overridden tls.Conn - ctx.c = &struct { - *tls.Conn - fooBar bool - }{} - if !ctx.IsTLS() { - t.Fatal("IsTLS must return true") - } - - ctx.c = &perIPConn{Conn: &tls.Conn{}} - if !ctx.IsTLS() { - t.Fatal("IsTLS must return true") - } -} - -func TestRequestCtxRedirectHTTPSSchemeless(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - - s := "GET /foo/bar?baz HTTP/1.1\nHost: aaa.com\n\n" - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := ctx.Request.Read(br); err != nil { - t.Fatalf("cannot read request: %v", err) - } - ctx.Request.isTLS = true - - ctx.Redirect("//foobar.com/aa/bbb", StatusFound) - location := ctx.Response.Header.Peek(HeaderLocation) - expectedLocation := "https://foobar.com/aa/bbb" - if string(location) != expectedLocation { - t.Fatalf("Unexpected location: %q. Expecting %q", location, expectedLocation) - } -} - -func TestRequestCtxRedirect(t *testing.T) { - t.Parallel() - - testRequestCtxRedirect(t, "http://qqq/", "", "http://qqq/") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "", "http://qqq/foo/bar?baz=111") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "#aaa", "http://qqq/foo/bar?baz=111#aaa") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "?abc=de&f", "http://qqq/foo/bar?abc=de&f") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "?abc=de&f#sf", "http://qqq/foo/bar?abc=de&f#sf") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "x.html", "http://qqq/foo/x.html") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "x.html?a=1", "http://qqq/foo/x.html?a=1") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "x.html#aaa=bbb&cc=ddd", "http://qqq/foo/x.html#aaa=bbb&cc=ddd") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "x.html?b=1#aaa=bbb&cc=ddd", "http://qqq/foo/x.html?b=1#aaa=bbb&cc=ddd") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "/x.html", "http://qqq/x.html") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "/x.html#aaa=bbb&cc=ddd", "http://qqq/x.html#aaa=bbb&cc=ddd") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "http://foo.bar/baz", "http://foo.bar/baz") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "https://foo.bar/baz", "https://foo.bar/baz") - testRequestCtxRedirect(t, "https://foo.com/bar?aaa", "//google.com/aaa?bb", "https://google.com/aaa?bb") - - if runtime.GOOS != "windows" { - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "../x.html", "http://qqq/x.html") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "../../x.html", "http://qqq/x.html") - testRequestCtxRedirect(t, "http://qqq/foo/bar?baz=111", "./.././../x.html", "http://qqq/x.html") - } -} - -func testRequestCtxRedirect(t *testing.T, origURL, redirectURL, expectedURL string) { - var ctx RequestCtx - var req Request - req.SetRequestURI(origURL) - ctx.Init(&req, nil, nil) - - ctx.Redirect(redirectURL, StatusFound) - loc := ctx.Response.Header.Peek(HeaderLocation) - if string(loc) != expectedURL { - t.Fatalf("unexpected redirect url %q. Expecting %q. origURL=%q, redirectURL=%q", loc, expectedURL, origURL, redirectURL) - } -} - -func TestServerResponseServerHeader(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - serverName := "foobar serv" - - s := &Server{ - Handler: func(ctx *RequestCtx) { - name := ctx.Response.Header.Server() - if string(name) != serverName { - fmt.Fprintf(ctx, "unexpected server name: %q. Expecting %q", name, serverName) - } else { - ctx.WriteString("OK") //nolint:errcheck - } - - // make sure the server name is sent to the client after ctx.Response.Reset() - ctx.NotFound() - }, - Name: serverName, - } - - ln := fasthttputil.NewInmemoryListener() - - serverCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverCh) - }() - - clientCh := make(chan struct{}) - go func() { - c, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if _, err = c.Write([]byte("GET / HTTP/1.1\r\nHost: aa\r\n\r\n")); err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(c) - var resp Response - if err = resp.Read(br); err != nil { - t.Errorf("unexpected error: %v", err) - } - - if resp.StatusCode() != StatusNotFound { - t.Errorf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusNotFound) - } - if string(resp.Body()) != "404 Page not found" { - t.Errorf("unexpected body: %q. Expecting %q", resp.Body(), "404 Page not found") - } - if string(resp.Header.Server()) != serverName { - t.Errorf("unexpected server header: %q. Expecting %q", resp.Header.Server(), serverName) - } - if err = c.Close(); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(clientCh) - }() - - select { - case <-clientCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } -} - -func TestServerResponseBodyStream(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - readyCh := make(chan struct{}) - h := func(ctx *RequestCtx) { - ctx.SetConnectionClose() - if ctx.IsBodyStream() { - t.Fatal("IsBodyStream must return false") - } - ctx.SetBodyStreamWriter(func(w *bufio.Writer) { - fmt.Fprintf(w, "first") - if err := w.Flush(); err != nil { - return - } - <-readyCh - fmt.Fprintf(w, "second") - // there is no need to flush w here, since it will - // be flushed automatically after returning from StreamWriter. - }) - if !ctx.IsBodyStream() { - t.Fatal("IsBodyStream must return true") - } - } - - serverCh := make(chan struct{}) - go func() { - if err := Serve(ln, h); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverCh) - }() - - clientCh := make(chan struct{}) - go func() { - c, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if _, err = c.Write([]byte("GET / HTTP/1.1\r\nHost: aa\r\n\r\n")); err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(c) - var respH ResponseHeader - if err = respH.Read(br); err != nil { - t.Errorf("unexpected error: %v", err) - } - if respH.StatusCode() != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d", respH.StatusCode(), StatusOK) - } - - buf := make([]byte, 1024) - n, err := br.Read(buf) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - b := buf[:n] - if string(b) != "5\r\nfirst\r\n" { - t.Errorf("unexpected result %q. Expecting %q", b, "5\r\nfirst\r\n") - } - close(readyCh) - - tail, err := ioutil.ReadAll(br) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if string(tail) != "6\r\nsecond\r\n0\r\n\r\n" { - t.Errorf("unexpected tail %q. Expecting %q", tail, "6\r\nsecond\r\n0\r\n\r\n") - } - - close(clientCh) - }() - - select { - case <-clientCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } -} - -func TestServerDisableKeepalive(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.WriteString("OK") //nolint:errcheck - }, - DisableKeepalive: true, - } - - ln := fasthttputil.NewInmemoryListener() - - serverCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverCh) - }() - - clientCh := make(chan struct{}) - go func() { - c, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if _, err = c.Write([]byte("GET / HTTP/1.1\r\nHost: aa\r\n\r\n")); err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(c) - var resp Response - if err = resp.Read(br); err != nil { - t.Errorf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - if !resp.ConnectionClose() { - t.Error("expecting 'Connection: close' response header") - } - if string(resp.Body()) != "OK" { - t.Errorf("unexpected body: %q. Expecting %q", resp.Body(), "OK") - } - - // make sure the connection is closed - data, err := ioutil.ReadAll(br) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if len(data) > 0 { - t.Errorf("unexpected data read from the connection: %q. Expecting empty data", data) - } - - close(clientCh) - }() - - select { - case <-clientCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } -} - -func TestServerMaxConnsPerIPLimit(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.WriteString("OK") //nolint:errcheck - }, - MaxConnsPerIP: 1, - Logger: &testLogger{}, - } - - ln := fasthttputil.NewInmemoryListener() - - serverCh := make(chan struct{}) - go func() { - fakeLN := &fakeIPListener{ - Listener: ln, - } - if err := s.Serve(fakeLN); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverCh) - }() - - clientCh := make(chan struct{}) - go func() { - c1, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - c2, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(c2) - var resp Response - if err = resp.Read(br); err != nil { - t.Errorf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusTooManyRequests { - t.Errorf("unexpected status code for the second connection: %d. Expecting %d", - resp.StatusCode(), StatusTooManyRequests) - } - - if _, err = c1.Write([]byte("GET / HTTP/1.1\r\nHost: aa\r\n\r\n")); err != nil { - t.Errorf("unexpected error when writing to the first connection: %v", err) - } - br = bufio.NewReader(c1) - if err = resp.Read(br); err != nil { - t.Errorf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code for the first connection: %d. Expecting %d", - resp.StatusCode(), StatusOK) - } - if string(resp.Body()) != "OK" { - t.Errorf("unexpected body for the first connection: %q. Expecting %q", resp.Body(), "OK") - } - close(clientCh) - }() - - select { - case <-clientCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } -} - -type fakeIPListener struct { - net.Listener -} - -func (ln *fakeIPListener) Accept() (net.Conn, error) { - conn, err := ln.Listener.Accept() - if err != nil { - return nil, err - } - return &fakeIPConn{ - Conn: conn, - }, nil -} - -type fakeIPConn struct { - net.Conn -} - -func (conn *fakeIPConn) RemoteAddr() net.Addr { - addr, err := net.ResolveTCPAddr("tcp4", "1.2.3.4:5789") - if err != nil { - panic(fmt.Sprintf("BUG: unexpected error: %v", err)) - } - return addr -} - -func TestServerConcurrencyLimit(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.WriteString("OK") //nolint:errcheck - }, - Concurrency: 1, - Logger: &testLogger{}, - } - - ln := fasthttputil.NewInmemoryListener() - - serverCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(serverCh) - }() - - clientCh := make(chan struct{}) - go func() { - c1, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - c2, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(c2) - var resp Response - if err = resp.Read(br); err != nil { - t.Errorf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusServiceUnavailable { - t.Errorf("unexpected status code for the second connection: %d. Expecting %d", - resp.StatusCode(), StatusServiceUnavailable) - } - - if _, err = c1.Write([]byte("GET / HTTP/1.1\r\nHost: aa\r\n\r\n")); err != nil { - t.Errorf("unexpected error when writing to the first connection: %v", err) - } - br = bufio.NewReader(c1) - if err = resp.Read(br); err != nil { - t.Errorf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code for the first connection: %d. Expecting %d", - resp.StatusCode(), StatusOK) - } - if string(resp.Body()) != "OK" { - t.Errorf("unexpected body for the first connection: %q. Expecting %q", resp.Body(), "OK") - } - close(clientCh) - }() - - select { - case <-clientCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } -} - -func TestServerWriteFastError(t *testing.T) { - t.Parallel() - - s := &Server{ - Name: "foobar", - } - var buf bytes.Buffer - expectedBody := "access denied" - s.writeFastError(&buf, StatusForbidden, expectedBody) - - br := bufio.NewReader(&buf) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if resp.StatusCode() != StatusForbidden { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusForbidden) - } - body := resp.Body() - if string(body) != expectedBody { - t.Fatalf("unexpected body: %q. Expecting %q", body, expectedBody) - } - server := string(resp.Header.Server()) - if server != s.Name { - t.Fatalf("unexpected server: %q. Expecting %q", server, s.Name) - } - contentType := string(resp.Header.ContentType()) - if contentType != "text/plain" { - t.Fatalf("unexpected content-type: %q. Expecting %q", contentType, "text/plain") - } - if !resp.Header.ConnectionClose() { - t.Fatal("expecting 'Connection: close' response header") - } -} - -func TestServerTLS(t *testing.T) { - t.Parallel() - - text := []byte("Make fasthttp great again") - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Write(text) //nolint:errcheck - }, - } - - certData, keyData, err := GenerateTestCertificate("localhost") - if err != nil { - t.Fatal(err) - } - - err = s.AppendCertEmbed(certData, keyData) - if err != nil { - t.Fatal(err) - } - go func() { - err = s.ServeTLS(ln, "", "") - if err != nil { - t.Error(err) - } - }() - - c := &Client{ - ReadTimeout: time.Second * 2, - Dial: func(addr string) (net.Conn, error) { - return ln.Dial() - }, - TLSConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - - req, res := defaultHTTPPool.AcquireRequest(), defaultHTTPPool.AcquireResponse() - req.SetRequestURI("https://some.url") - - err = c.Do(req, res) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(text, res.Body()) { - t.Fatal("error transmitting information") - } -} - -func TestServerTLSReadTimeout(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - ReadTimeout: time.Millisecond * 500, - Logger: &testLogger{}, // Ignore log output. - Handler: func(ctx *RequestCtx) { - }, - } - - certData, keyData, err := GenerateTestCertificate("localhost") - if err != nil { - t.Fatal(err) - } - - err = s.AppendCertEmbed(certData, keyData) - if err != nil { - t.Fatal(err) - } - go func() { - err = s.ServeTLS(ln, "", "") - if err != nil { - t.Error(err) - } - }() - - c, err := ln.Dial() - if err != nil { - t.Error(err) - } - - r := make(chan error) - - go func() { - b := make([]byte, 1) - _, err := c.Read(b) - c.Close() - r <- err - }() - - select { - case err = <-r: - case <-time.After(time.Second * 2): - } - - if err == nil { - t.Error("server didn't close connection after timeout") - } -} - -func TestServerServeTLSEmbed(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - certData, keyData, err := GenerateTestCertificate("localhost") - if err != nil { - t.Fatal(err) - } - - // start the server - ch := make(chan struct{}) - go func() { - err := ServeTLSEmbed(ln, certData, keyData, func(ctx *RequestCtx) { - if !ctx.IsTLS() { - ctx.Error("expecting tls", StatusBadRequest) - return - } - if !ctx.PhantomURI().isHttps() { - ctx.Error(fmt.Sprintf("unexpected scheme=%q. Expecting %q", ctx.PhantomURI().Scheme(), "https"), StatusBadRequest) - return - } - ctx.WriteString("success") //nolint:errcheck - }) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - close(ch) - }() - - // establish connection to the server - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - tlsConn := tls.Client(conn, &tls.Config{ - InsecureSkipVerify: true, - }) - - // send request - if _, err = tlsConn.Write([]byte("GET / HTTP/1.1\r\nHost: aaa\r\n\r\n")); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // read response - respCh := make(chan struct{}) - go func() { - br := bufio.NewReader(tlsConn) - var resp Response - if err := resp.Read(br); err != nil { - t.Error("unexpected error") - } - body := resp.Body() - if string(body) != "success" { - t.Errorf("unexpected response body %q. Expecting %q", body, "success") - } - close(respCh) - }() - select { - case <-respCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - // close the server - if err = ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout") - } -} - -func TestServerMultipartFormDataRequest(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - for _, test := range []struct { - StreamRequestBody bool - DisablePreParseMultipartForm bool - }{ - {false, false}, - {false, true}, - {true, false}, - {true, true}, - } { - reqS := `POST /upload HTTP/1.1 -Host: qwerty.com -Content-Length: 521 -Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryJwfATyF8tmxSJnLg - -------WebKitFormBoundaryJwfATyF8tmxSJnLg -Content-Disposition: form-data; name="f1" - -value1 -------WebKitFormBoundaryJwfATyF8tmxSJnLg -Content-Disposition: form-data; name="fileaaa"; filename="TODO" -Content-Type: application/octet-stream - -- SessionClient with referer and cookies support. -- Client with requests' pipelining support. -- ProxyHandler similar to FSHandler. -- WebSockets. See https://tools.ietf.org/html/rfc6455 . -- HTTP/2.0. See https://tools.ietf.org/html/rfc7540 . - -------WebKitFormBoundaryJwfATyF8tmxSJnLg-- - -GET / HTTP/1.1 -Host: asbd -Connection: close - -` - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - StreamRequestBody: test.StreamRequestBody, - DisablePreParseMultipartForm: test.DisablePreParseMultipartForm, - Handler: func(ctx *RequestCtx) { - switch string(ctx.Path()) { - case "/upload": - f, err := ctx.MultipartForm() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if len(f.Value) != 1 { - t.Errorf("unexpected values %d. Expecting %d", len(f.Value), 1) - } - if len(f.File) != 1 { - t.Errorf("unexpected file values %d. Expecting %d", len(f.File), 1) - } - fv := ctx.FormValue("f1") - if string(fv) != "value1" { - t.Errorf("unexpected form value: %q. Expecting %q", fv, "value1") - } - ctx.Redirect("/", StatusSeeOther) - default: - ctx.WriteString("non-upload") //nolint:errcheck - } - }, - } - - ch := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(ch) - }() - - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if _, err = conn.Write([]byte(reqS)); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var resp Response - br := bufio.NewReader(conn) - respCh := make(chan struct{}) - go func() { - if err := resp.Read(br); err != nil { - t.Errorf("error when reading response: %v", err) - } - if resp.StatusCode() != StatusSeeOther { - t.Errorf("unexpected status code %d. Expecting %d", resp.StatusCode(), StatusSeeOther) - } - loc := resp.Header.Peek(HeaderLocation) - if string(loc) != "http://qwerty.com/" { - t.Errorf("unexpected location %q. Expecting %q", loc, "http://qwerty.com/") - } - - if err := resp.Read(br); err != nil { - t.Errorf("error when reading the second response: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - body := resp.Body() - if string(body) != "non-upload" { - t.Errorf("unexpected body %q. Expecting %q", body, "non-upload") - } - close(respCh) - }() - - select { - case <-respCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if err := ln.Close(); err != nil { - t.Fatalf("error when closing listener: %v", err) - } - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout when waiting for the server to stop") - } - } -} - -func TestServerGetWithContent(t *testing.T) { - t.Parallel() - - h := func(ctx *RequestCtx) { - ctx.Success("foo/bar", []byte("success")) - } - s := &Server{ - Handler: h, - } - - rw := &readWriter{} - rw.r.WriteString("GET / HTTP/1.1\r\nHost: mm.com\r\nContent-Length: 5\r\n\r\nabcde") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - resp := rw.w.String() - if !strings.HasSuffix(resp, "success") { - t.Fatalf("unexpected response %q.", resp) - } -} - -func TestServerDisableHeaderNamesNormalizing(t *testing.T) { - t.Parallel() - - headerName := "CASE-senSITive-HEAder-NAME" - headerNameLower := strings.ToLower(headerName) - headerValue := "foobar baz" - s := &Server{ - Handler: func(ctx *RequestCtx) { - hv := ctx.Request.Header.Peek(headerName) - if string(hv) != headerValue { - t.Errorf("unexpected header value for %q: %q. Expecting %q", headerName, hv, headerValue) - } - hv = ctx.Request.Header.Peek(headerNameLower) - if len(hv) > 0 { - t.Errorf("unexpected header value for %q: %q. Expecting empty value", headerNameLower, hv) - } - ctx.Response.Header.Set(headerName, headerValue) - ctx.WriteString("ok") //nolint:errcheck - ctx.SetContentType("aaa") - }, - DisableHeaderNamesNormalizing: true, - } - - rw := &readWriter{} - rw.r.WriteString(fmt.Sprintf("GET / HTTP/1.1\r\n%s: %s\r\nHost: google.com\r\n\r\n", headerName, headerValue)) - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - var resp Response - resp.Header.DisableNormalizing() - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - hv := resp.Header.Peek(headerName) - if string(hv) != headerValue { - t.Fatalf("unexpected header value for %q: %q. Expecting %q", headerName, hv, headerValue) - } - hv = resp.Header.Peek(headerNameLower) - if len(hv) > 0 { - t.Fatalf("unexpected header value for %q: %q. Expecting empty value", headerNameLower, hv) - } -} - -func TestServerReduceMemoryUsageSerial(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) {}, - ReduceMemoryUsage: true, - } - - ch := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(ch) - }() - - testServerRequests(t, ln) - - if err := ln.Close(); err != nil { - t.Fatalf("error when closing listener: %v", err) - } - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout when waiting for the server to stop") - } -} - -func TestServerReduceMemoryUsageConcurrent(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - Handler: func(ctx *RequestCtx) {}, - ReduceMemoryUsage: true, - } - - ch := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(ch) - }() - - gCh := make(chan struct{}) - for i := 0; i < 10; i++ { - go func() { - testServerRequests(t, ln) - gCh <- struct{}{} - }() - } - for i := 0; i < 10; i++ { - select { - case <-gCh: - case <-time.After(time.Second): - t.Fatalf("timeout on goroutine %d", i) - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("error when closing listener: %v", err) - } - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout when waiting for the server to stop") - } -} - -func testServerRequests(t *testing.T, ln *fasthttputil.InmemoryListener) { - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - br := bufio.NewReader(conn) - var resp Response - for i := 0; i < 10; i++ { - if _, err = fmt.Fprintf(conn, "GET / HTTP/1.1\r\nHost: aaa\r\n\r\n"); err != nil { - t.Fatalf("unexpected error on iteration %d: %v", i, err) - } - - respCh := make(chan struct{}) - go func() { - if err = resp.Read(br); err != nil { - t.Errorf("unexpected error when reading response on iteration %d: %v", i, err) - } - close(respCh) - }() - select { - case <-respCh: - case <-time.After(time.Second): - t.Fatalf("timeout on iteration %d", i) - } - } - - if err = conn.Close(); err != nil { - t.Fatalf("error when closing the connection: %v", err) - } -} - -func TestServerHTTP10ConnectionKeepAlive(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - ch := make(chan struct{}) - go func() { - err := Serve(ln, func(ctx *RequestCtx) { - if string(ctx.Path()) == "/close" { - ctx.SetConnectionClose() - } - }) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - close(ch) - }() - - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - _, err = fmt.Fprintf(conn, "%s", "GET / HTTP/1.0\r\nHost: aaa\r\nConnection: keep-alive\r\n\r\n") - if err != nil { - t.Fatalf("error when writing request: %v", err) - } - _, err = fmt.Fprintf(conn, "%s", "GET /close HTTP/1.0\r\nHost: aaa\r\nConnection: keep-alive\r\n\r\n") - if err != nil { - t.Fatalf("error when writing request: %v", err) - } - - br := bufio.NewReader(conn) - var resp Response - if err = resp.Read(br); err != nil { - t.Fatalf("error when reading response: %v", err) - } - if resp.ConnectionClose() { - t.Fatal("response mustn't have 'Connection: close' header") - } - if err = resp.Read(br); err != nil { - t.Fatalf("error when reading response: %v", err) - } - if !resp.ConnectionClose() { - t.Fatal("response must have 'Connection: close' header") - } - - tailCh := make(chan struct{}) - go func() { - tail, err := ioutil.ReadAll(br) - if err != nil { - t.Errorf("error when reading tail: %v", err) - } - if len(tail) > 0 { - t.Errorf("unexpected non-zero tail %q", tail) - } - close(tailCh) - }() - - select { - case <-tailCh: - case <-time.After(time.Second): - t.Fatal("timeout when reading tail") - } - - if err = conn.Close(); err != nil { - t.Fatalf("error when closing the connection: %v", err) - } - - if err = ln.Close(); err != nil { - t.Fatalf("error when closing listener: %v", err) - } - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout when waiting for the server to stop") - } -} - -func TestServerHTTP10ConnectionClose(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - - ch := make(chan struct{}) - go func() { - err := Serve(ln, func(ctx *RequestCtx) { - // The server must close the connection irregardless - // of request and response state set inside request - // handler, since the HTTP/1.0 request - // had no 'Connection: keep-alive' header. - ctx.Request.Header.ResetConnectionClose() - ctx.Request.Header.Set(HeaderConnection, "keep-alive") - ctx.Response.Header.ResetConnectionClose() - ctx.Response.Header.Set(HeaderConnection, "keep-alive") - }) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - close(ch) - }() - - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - _, err = fmt.Fprintf(conn, "%s", "GET / HTTP/1.0\r\nHost: aaa\r\n\r\n") - if err != nil { - t.Fatalf("error when writing request: %v", err) - } - - br := bufio.NewReader(conn) - var resp Response - if err = resp.Read(br); err != nil { - t.Fatalf("error when reading response: %v", err) - } - - if !resp.ConnectionClose() { - t.Fatal("HTTP1.0 response must have 'Connection: close' header") - } - - tailCh := make(chan struct{}) - go func() { - tail, err := ioutil.ReadAll(br) - if err != nil { - t.Errorf("error when reading tail: %v", err) - } - if len(tail) > 0 { - t.Errorf("unexpected non-zero tail %q", tail) - } - close(tailCh) - }() - - select { - case <-tailCh: - case <-time.After(time.Second): - t.Fatal("timeout when reading tail") - } - - if err = conn.Close(); err != nil { - t.Fatalf("error when closing the connection: %v", err) - } - - if err = ln.Close(); err != nil { - t.Fatalf("error when closing listener: %v", err) - } - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout when waiting for the server to stop") - } -} - -func TestRequestCtxFormValue(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - var req Request - req.SetRequestURI("/foo/bar?baz=123&aaa=bbb") - req.SetBodyString("qqq=port&mmm=sddd") - req.Header.SetContentType("application/x-www-form-urlencoded") - - ctx.Init(&req, nil, nil) - - v := ctx.FormValue("baz") - if string(v) != "123" { - t.Fatalf("unexpected value %q. Expecting %q", v, "123") - } - v = ctx.FormValue("mmm") - if string(v) != "sddd" { - t.Fatalf("unexpected value %q. Expecting %q", v, "sddd") - } - v = ctx.FormValue("aaaasdfsdf") - if len(v) > 0 { - t.Fatalf("unexpected value for unknown key %q", v) - } -} - -func TestRequestCtxUserValue(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - - for i := 0; i < 5; i++ { - k := fmt.Sprintf("key-%d", i) - ctx.SetUserValue(k, i) - } - for i := 5; i < 10; i++ { - k := fmt.Sprintf("key-%d", i) - ctx.SetUserValueBytes([]byte(k), i) - } - - for i := 0; i < 10; i++ { - k := fmt.Sprintf("key-%d", i) - v := ctx.UserValue(k) - n, ok := v.(int) - if !ok || n != i { - t.Fatalf("unexpected value obtained for key %q: %v. Expecting %d", k, v, i) - } - } - vlen := 0 - ctx.VisitUserValues(func(key []byte, value interface{}) { - vlen++ - v := ctx.UserValueBytes(key) - if v != value { - t.Fatalf("unexpected value obtained from VisitUserValues for key: %q, expecting: %#v but got: %#v", key, v, value) - } - }) - if len(ctx.userValues) != vlen { - t.Fatalf("the length of user values returned from VisitUserValues is not equal to the length of the userValues, expecting: %d but got: %d", len(ctx.userValues), vlen) - } - - ctx.ResetUserValues() - for i := 0; i < 10; i++ { - k := fmt.Sprintf("key-%d", i) - v := ctx.UserValue(k) - if v != nil { - t.Fatalf("unexpected value obtained for key %q: %v. Expecting nil", k, v) - } - } -} - -func TestServerHeadRequest(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - fmt.Fprintf(ctx, "Request method is %q", ctx.Method()) - ctx.SetContentType("aaa/bbb") - }, - } - - rw := &readWriter{} - rw.r.WriteString("HEAD /foobar HTTP/1.1\r\nHost: aaa.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - var resp Response - resp.SkipBody = true - if err := resp.Read(br); err != nil { - t.Fatalf("Unexpected error when parsing response: %v", err) - } - if resp.Header.StatusCode() != StatusOK { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.Header.StatusCode(), StatusOK) - } - if len(resp.Body()) > 0 { - t.Fatalf("Unexpected non-zero body %q", resp.Body()) - } - if resp.Header.ContentLength() != 24 { - t.Fatalf("unexpected content-length %d. Expecting %d", resp.Header.ContentLength(), 24) - } - if string(resp.Header.ContentType()) != "aaa/bbb" { - t.Fatalf("unexpected content-type %q. Expecting %q", resp.Header.ContentType(), "aaa/bbb") - } - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("Unexpected error when reading remaining data: %v", err) - } - if len(data) > 0 { - t.Fatalf("unexpected remaining data %q", data) - } -} - -func TestServerExpect100Continue(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - if !ctx.IsPost() { - t.Errorf("unexpected method %q. Expecting POST", ctx.Method()) - } - if string(ctx.Path()) != "/foo" { - t.Errorf("unexpected path %q. Expecting %q", ctx.Path(), "/foo") - } - ct := ctx.Request.Header.ContentType() - if string(ct) != "a/b" { - t.Errorf("unexpectected content-type: %q. Expecting %q", ct, "a/b") - } - if string(ctx.PostBody()) != "12345" { - t.Errorf("unexpected body: %q. Expecting %q", ctx.PostBody(), "12345") - } - ctx.WriteString("foobar") //nolint:errcheck - }, - } - - rw := &readWriter{} - rw.r.WriteString("POST /foo HTTP/1.1\r\nHost: gle.com\r\nExpect: 100-continue\r\nContent-Length: 5\r\nContent-Type: a/b\r\n\r\n12345") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, StatusOK, string(defaultContentType), "foobar") - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("Unexpected error when reading remaining data: %v", err) - } - if len(data) > 0 { - t.Fatalf("unexpected remaining data %q", data) - } -} - -func TestServerContinueHandler(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - acceptContentLength := 5 - s := &Server{ - ContinueHandler: func(headers *RequestHeader) bool { - if !headers.IsPost() { - t.Errorf("unexpected method %q. Expecting POST", headers.Method()) - } - - ct := headers.ContentType() - if string(ct) != "a/b" { - t.Errorf("unexpectected content-type: %q. Expecting %q", ct, "a/b") - } - - // Pass on any request that isn't the accepted content length - return headers.contentLength == acceptContentLength - }, - Handler: func(ctx *RequestCtx) { - if ctx.Request.Header.contentLength != acceptContentLength { - t.Errorf("all requests with content-length: other than %d, should be denied", acceptContentLength) - } - if !ctx.IsPost() { - t.Errorf("unexpected method %q. Expecting POST", ctx.Method()) - } - if string(ctx.Path()) != "/foo" { - t.Errorf("unexpected path %q. Expecting %q", ctx.Path(), "/foo") - } - ct := ctx.Request.Header.ContentType() - if string(ct) != "a/b" { - t.Errorf("unexpectected content-type: %q. Expecting %q", ct, "a/b") - } - if string(ctx.PostBody()) != "12345" { - t.Errorf("unexpected body: %q. Expecting %q", ctx.PostBody(), "12345") - } - ctx.WriteString("foobar") //nolint:errcheck - }, - } - - sendRequest := func(rw *readWriter, expectedStatusCode int, expectedResponse string) { - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, expectedStatusCode, string(defaultContentType), expectedResponse) - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("Unexpected error when reading remaining data: %v", err) - } - if len(data) > 0 { - t.Fatalf("unexpected remaining data %q", data) - } - } - - // The same server should not fail when handling the three different types of requests - // Regular requests - // Expect 100 continue accepted - // Exepect 100 continue denied - rw := &readWriter{} - for i := 0; i < 25; i++ { - - // Regular requests without Expect 100 continue header - rw.r.Reset() - rw.r.WriteString("POST /foo HTTP/1.1\r\nHost: gle.com\r\nContent-Length: 5\r\nContent-Type: a/b\r\n\r\n12345") - sendRequest(rw, StatusOK, "foobar") - - // Regular Expect 100 continue reqeuests that are accepted - rw.r.Reset() - rw.r.WriteString("POST /foo HTTP/1.1\r\nHost: gle.com\r\nExpect: 100-continue\r\nContent-Length: 5\r\nContent-Type: a/b\r\n\r\n12345") - sendRequest(rw, StatusOK, "foobar") - - // Requests being denied - rw.r.Reset() - rw.r.WriteString("POST /foo HTTP/1.1\r\nHost: gle.com\r\nExpect: 100-continue\r\nContent-Length: 6\r\nContent-Type: a/b\r\n\r\n123456") - sendRequest(rw, StatusExpectationFailed, "") - } -} - -func TestCompressHandler(t *testing.T) { - t.Parallel() - - expectedBody := string(createFixedBody(2e4)) - h := CompressHandler(func(ctx *RequestCtx) { - ctx.Write([]byte(expectedBody)) //nolint:errcheck - }) - - var ctx RequestCtx - var resp Response - - // verify uncompressed response - h(&ctx) - s := ctx.Response.String() - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - ce := resp.Header.ContentEncoding() - if string(ce) != "" { - t.Fatalf("unexpected Content-Encoding: %q. Expecting %q", ce, "") - } - body := resp.Body() - if string(body) != expectedBody { - t.Fatalf("unexpected body %q. Expecting %q", body, expectedBody) - } - - // verify gzip-compressed response - ctx.Request.Reset() - ctx.Response.Reset() - ctx.Request.Header.Set("Accept-Encoding", "gzip, deflate, sdhc") - - h(&ctx) - s = ctx.Response.String() - br = bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - ce = resp.Header.ContentEncoding() - if string(ce) != "gzip" { - t.Fatalf("unexpected Content-Encoding: %q. Expecting %q", ce, "gzip") - } - body, err := resp.BodyGunzip() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(body) != expectedBody { - t.Fatalf("unexpected body %q. Expecting %q", body, expectedBody) - } - - // an attempt to compress already compressed response - ctx.Request.Reset() - ctx.Response.Reset() - ctx.Request.Header.Set("Accept-Encoding", "gzip, deflate, sdhc") - hh := CompressHandler(h) - hh(&ctx) - s = ctx.Response.String() - br = bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - ce = resp.Header.ContentEncoding() - if string(ce) != "gzip" { - t.Fatalf("unexpected Content-Encoding: %q. Expecting %q", ce, "gzip") - } - body, err = resp.BodyGunzip() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(body) != expectedBody { - t.Fatalf("unexpected body %q. Expecting %q", body, expectedBody) - } - - // verify deflate-compressed response - ctx.Request.Reset() - ctx.Response.Reset() - ctx.Request.Header.Set(HeaderAcceptEncoding, "foobar, deflate, sdhc") - - h(&ctx) - s = ctx.Response.String() - br = bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - ce = resp.Header.ContentEncoding() - if string(ce) != "deflate" { - t.Fatalf("unexpected Content-Encoding: %q. Expecting %q", ce, "deflate") - } - body, err = resp.BodyInflate() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if string(body) != expectedBody { - t.Fatalf("unexpected body %q. Expecting %q", body, expectedBody) - } -} - -func TestRequestCtxWriteString(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - n, err := ctx.WriteString("foo") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if n != 3 { - t.Fatalf("unexpected n %d. Expecting 3", n) - } - n, err = ctx.WriteString("привет") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if n != 12 { - t.Fatalf("unexpected n=%d. Expecting 12", n) - } - - s := ctx.Response.Body() - if string(s) != "fooпривет" { - t.Fatalf("unexpected response body %q. Expecting %q", s, "fooпривет") - } -} - -func TestServeConnKeepRequestAndResponseUntilResetUserValues(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - reqStr := "POST /foo HTTP/1.0\r\nHost: google.com\r\nContent-Type: application/octet-stream\r\nContent-Length: 0\r\nConnection: keep-alive\r\n\r\n" - respRegex := regexp.MustCompile("HTTP/1.1 308 Permanent Redirect\r\nServer: fasthttp\r\nDate: (.*)\r\nContent-Length: 0\r\nConnection: keep-alive\r\n\r\n") - - rw := &readWriter{} - rw.r.WriteString(reqStr) - - var resultReqStr, resultRespStr string - - ch := make(chan struct{}) - go func() { - err := ServeConn(rw, func(ctx *RequestCtx) { - ctx.Response.SetStatusCode(StatusPermanentRedirect) - - ctx.SetUserValue("myKey", &closerWithRequestCtx{ - ctx: ctx, - closeFunc: func(closerCtx *RequestCtx) error { - resultReqStr = closerCtx.Request.String() - resultRespStr = closerCtx.Response.String() - - return nil - }}) - }) - if err != nil { - t.Errorf("unexpected error in ServeConn: %v", err) - } - close(ch) - }() - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if resultReqStr != reqStr { - t.Errorf("Request == %q, want %q", resultReqStr, reqStr) - } - - if !respRegex.MatchString(resultRespStr) { - t.Errorf("Response == %q, want regex %q", resultRespStr, respRegex) - } -} - -// TestServerErrorHandler tests unexpected cases the for loop will break -// before request/response reset call. in such cases, call it before -// release to fix #548. -func TestServerErrorHandler(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var resultReqStr, resultRespStr string - - s := &Server{ - Handler: func(ctx *RequestCtx) {}, - ErrorHandler: func(ctx *RequestCtx, _ error) { - resultReqStr = ctx.Request.String() - resultRespStr = ctx.Response.String() - }, - MaxRequestBodySize: 10, - } - - reqStrTpl := "POST %s HTTP/1.1\r\nHost: example.com\r\nContent-Type: application/octet-stream\r\nContent-Length: %d\r\nConnection: keep-alive\r\n\r\n" - respRegex := regexp.MustCompile("HTTP/1.1 200 OK\r\nDate: (.*)\r\nContent-Length: 0\r\n\r\n") - - rw := &readWriter{} - - for i := 0; i < 100; i++ { - body := strings.Repeat("@", s.MaxRequestBodySize+1) - path := fmt.Sprintf("/%d", i) - - reqStr := fmt.Sprintf(reqStrTpl, path, len(body)) - expectedReqStr := fmt.Sprintf(reqStrTpl, path, 0) - - rw.r.WriteString(reqStr) - rw.r.WriteString(body) - - ch := make(chan struct{}) - go func() { - err := s.ServeConn(rw) - if err != nil && !errors.Is(err, ErrBodyTooLarge) { - t.Errorf("unexpected error in ServeConn: %v", err) - } - close(ch) - }() - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if resultReqStr != expectedReqStr { - t.Errorf("[iter: %d] Request == %q, want %s", i, resultReqStr, reqStr) - } - - if !respRegex.MatchString(resultRespStr) { - t.Errorf("[iter: %d] Response == %q, want regex %q", i, resultRespStr, respRegex) - } - } -} - -func TestServeConnHijackResetUserValues(t *testing.T) { - t.Parallel() - - rw := &readWriter{} - rw.r.WriteString("GET /foo HTTP/1.0\r\nConnection: keep-alive\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("") - - ch := make(chan struct{}) - go func() { - err := ServeConn(rw, func(ctx *RequestCtx) { - ctx.Hijack(func(c net.Conn) {}) - ctx.SetUserValue("myKey", &closerWithRequestCtx{ - closeFunc: func(_ *RequestCtx) error { - close(ch) - - return nil - }}, - ) - }) - if err != nil { - t.Errorf("unexpected error in ServeConn: %v", err) - } - }() - - select { - case <-ch: - case <-time.After(time.Second): - t.Errorf("Timeout: UserValues should be reset") - } -} - -func TestServeConnNonHTTP11KeepAlive(t *testing.T) { - t.Parallel() - - rw := &readWriter{} - rw.r.WriteString("GET /foo HTTP/1.0\r\nConnection: keep-alive\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("GET /bar HTTP/1.0\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("GET /must/be/ignored HTTP/1.0\r\nHost: google.com\r\n\r\n") - - requestsServed := 0 - - ch := make(chan struct{}) - go func() { - err := ServeConn(rw, func(ctx *RequestCtx) { - requestsServed++ - ctx.SuccessString("aaa/bbb", "foobar") - }) - if err != nil { - t.Errorf("unexpected error in ServeConn: %v", err) - } - close(ch) - }() - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - br := bufio.NewReader(&rw.w) - - var resp Response - - // verify the first response - if err := resp.Read(br); err != nil { - t.Fatalf("Unexpected error when parsing response: %v", err) - } - if string(resp.Header.Peek(HeaderConnection)) != "keep-alive" { - t.Fatalf("unexpected Connection header %q. Expecting %q", resp.Header.Peek(HeaderConnection), "keep-alive") - } - if resp.Header.ConnectionClose() { - t.Fatal("unexpected Connection: close") - } - - // verify the second response - if err := resp.Read(br); err != nil { - t.Fatalf("Unexpected error when parsing response: %v", err) - } - if string(resp.Header.Peek(HeaderConnection)) != "close" { - t.Fatalf("unexpected Connection header %q. Expecting %q", resp.Header.Peek(HeaderConnection), "close") - } - if !resp.Header.ConnectionClose() { - t.Fatal("expecting Connection: close") - } - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("Unexpected error when reading remaining data: %v", err) - } - if len(data) != 0 { - t.Fatalf("Unexpected data read after responses %q", data) - } - - if requestsServed != 2 { - t.Fatalf("unexpected number of requests served: %d. Expecting 2", requestsServed) - } -} - -func TestRequestCtxSetBodyStreamWriter(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - var req Request - ctx.Init(&req, nil, nil) - - if ctx.IsBodyStream() { - t.Fatal("IsBodyStream must return false") - } - ctx.SetBodyStreamWriter(func(w *bufio.Writer) { - fmt.Fprintf(w, "body writer line 1\n") - if err := w.Flush(); err != nil { - t.Errorf("unexpected error: %v", err) - } - fmt.Fprintf(w, "body writer line 2\n") - }) - if !ctx.IsBodyStream() { - t.Fatal("IsBodyStream must return true") - } - - s := ctx.Response.String() - - br := bufio.NewReader(bytes.NewBufferString(s)) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("Error when reading response: %v", err) - } - - body := string(resp.Body()) - expectedBody := "body writer line 1\nbody writer line 2\n" - if body != expectedBody { - t.Fatalf("unexpected body: %q. Expecting %q", body, expectedBody) - } -} - -func TestRequestCtxIfModifiedSince(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - var req Request - ctx.Init(&req, nil, nil) - - lastModified := time.Now().Add(-time.Hour) - - if !ctx.IfModifiedSince(lastModified) { - t.Fatal("IfModifiedSince must return true for non-existing If-Modified-Since header") - } - - ctx.Request.Header.Set("If-Modified-Since", string(AppendHTTPDate(nil, lastModified))) - - if ctx.IfModifiedSince(lastModified) { - t.Fatal("If-Modified-Since current time must return false") - } - - past := lastModified.Add(-time.Hour) - if ctx.IfModifiedSince(past) { - t.Fatal("If-Modified-Since past time must return false") - } - - future := lastModified.Add(time.Hour) - if !ctx.IfModifiedSince(future) { - t.Fatal("If-Modified-Since future time must return true") - } -} - -func TestRequestCtxSendFileNotModified(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - var req Request - ctx.Init(&req, nil, nil) - - filePath := "./server_test.go" - lastModified, err := FileLastModified(filePath) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - ctx.Request.Header.Set("If-Modified-Since", string(AppendHTTPDate(nil, lastModified))) - - ctx.SendFile(filePath) - - s := ctx.Response.String() - - var resp Response - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("error when reading response: %v", err) - } - if resp.StatusCode() != StatusNotModified { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusNotModified) - } - if len(resp.Body()) > 0 { - t.Fatalf("unexpected non-zero response body: %q", resp.Body()) - } -} - -func TestRequestCtxSendFileModified(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - var req Request - ctx.Init(&req, nil, nil) - - filePath := "./server_test.go" - lastModified, err := FileLastModified(filePath) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - lastModified = lastModified.Add(-time.Hour) - ctx.Request.Header.Set("If-Modified-Since", string(AppendHTTPDate(nil, lastModified))) - - ctx.SendFile(filePath) - - s := ctx.Response.String() - - var resp Response - br := bufio.NewReader(bytes.NewBufferString(s)) - if err := resp.Read(br); err != nil { - t.Fatalf("error when reading response: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - - f, err := os.Open(filePath) - if err != nil { - t.Fatalf("cannot open file: %v", err) - } - body, err := ioutil.ReadAll(f) - f.Close() - if err != nil { - t.Fatalf("error when reading file: %v", err) - } - - if !bytes.Equal(resp.Body(), body) { - t.Fatalf("unexpected response body: %q. Expecting %q", resp.Body(), body) - } -} - -func TestRequestCtxSendFile(t *testing.T) { - t.Parallel() - - var ctx RequestCtx - var req Request - ctx.Init(&req, nil, nil) - - filePath := "./server_test.go" - ctx.SendFile(filePath) - - w := &bytes.Buffer{} - bw := bufio.NewWriter(w) - if err := ctx.Response.Write(bw); err != nil { - t.Fatalf("error when writing response: %v", err) - } - if err := bw.Flush(); err != nil { - t.Fatalf("error when flushing response: %v", err) - } - - var resp Response - br := bufio.NewReader(w) - if err := resp.Read(br); err != nil { - t.Fatalf("error when reading response: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) - } - - f, err := os.Open(filePath) - if err != nil { - t.Fatalf("cannot open file: %v", err) - } - body, err := ioutil.ReadAll(f) - f.Close() - if err != nil { - t.Fatalf("error when reading file: %v", err) - } - - if !bytes.Equal(resp.Body(), body) { - t.Fatalf("unexpected response body: %q. Expecting %q", resp.Body(), body) - } -} - -func testRequestCtxHijack(t *testing.T, s *Server) { - t.Helper() - - type hijackSignal struct { - id int - rw *readWriter - } - - wg := sync.WaitGroup{} - totalConns := 100 - hijackStartCh := make(chan *hijackSignal, totalConns) - hijackStopCh := make(chan *hijackSignal, totalConns) - - s.Handler = func(ctx *RequestCtx) { - if ctx.Hijacked() { - t.Error("connection mustn't be hijacked") - } - - ctx.Hijack(func(c net.Conn) { - signal := <-hijackStartCh - defer func() { - hijackStopCh <- signal - wg.Done() - }() - - b := make([]byte, 1) - stop := false - - // ping-pong echo via hijacked conn - for !stop { - n, err := c.Read(b) - if err != nil { - if errors.Is(err, io.EOF) { - stop = true - - continue - } - - t.Errorf("unexpected read error: %v", err) - } else if n != 1 { - t.Errorf("unexpected number of bytes read: %d. Expecting 1", n) - } - - if _, err = c.Write(b); err != nil { - t.Errorf("unexpected error when writing data: %v", err) - } - } - }) - - if !ctx.Hijacked() { - t.Error("connection must be hijacked") - } - - ctx.Success("foo/bar", []byte("hijack it!")) - } - - hijackedString := "foobar baz hijacked!!!" - - for i := 0; i < totalConns; i++ { - wg.Add(1) - - go func(t *testing.T, id int) { - t.Helper() - - rw := new(readWriter) - rw.r.WriteString("GET /foo HTTP/1.1\r\nHost: google.com\r\n\r\n") - rw.r.WriteString(hijackedString) - - if err := s.ServeConn(rw); err != nil { - t.Errorf("[iter: %d] Unexpected error from serveConn: %v", id, err) - } - - hijackStartCh <- &hijackSignal{id, rw} - }(t, i) - } - - wg.Wait() - - count := 0 - for count != totalConns { - select { - case signal := <-hijackStopCh: - count++ - - id := signal.id - rw := signal.rw - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, StatusOK, "foo/bar", "hijack it!") - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Errorf("[iter: %d] Unexpected error when reading remaining data: %v", id, err) - - return - } - if string(data) != hijackedString { - t.Errorf( - "[iter: %d] Unexpected response %q. Expecting %q", - id, data, hijackedString, - ) - - return - } - case <-time.After(200 * time.Millisecond): - t.Errorf("timeout") - } - } - - close(hijackStartCh) - close(hijackStopCh) -} - -func TestRequestCtxHijack(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - testRequestCtxHijack(t, &Server{}) -} - -func TestRequestCtxHijackReduceMemoryUsage(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - testRequestCtxHijack(t, &Server{ - ReduceMemoryUsage: true, - }) -} - -func TestRequestCtxHijackNoResponse(t *testing.T) { - t.Parallel() - - hijackDone := make(chan error) - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Hijack(func(c net.Conn) { - _, err := c.Write([]byte("test")) - hijackDone <- err - }) - ctx.HijackSetNoResponse(true) - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo HTTP/1.1\r\nHost: google.com\r\nContent-Length: 0\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - select { - case err := <-hijackDone: - if err != nil { - t.Fatalf("Unexpected error from hijack: %v", err) - } - case <-time.After(100 * time.Millisecond): - t.Fatal("timeout") - } - - if got := rw.w.String(); got != "test" { - t.Errorf(`expected "test", got %q`, got) - } -} - -func TestRequestCtxNoHijackNoResponse(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - io.WriteString(ctx, "test") //nolint:errcheck - ctx.HijackSetNoResponse(true) - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo HTTP/1.1\r\nHost: google.com\r\nContent-Length: 0\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - bf := bufio.NewReader( - strings.NewReader(rw.w.String()), - ) - resp := defaultHTTPPool.AcquireResponse() - resp.Read(bf) //nolint:errcheck - if got := string(resp.Body()); got != "test" { - t.Errorf(`expected "test", got %q`, got) - } -} - -func TestRequestCtxInit(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - // This test can't run parallel as it modifies globalConnID. - - var ctx RequestCtx - var logger testLogger - globalConnID = 0x123456 - ctx.Init(&ctx.Request, zeroTCPAddr, &logger) - ip := ctx.RemoteIP() - if !ip.IsUnspecified() { - t.Fatalf("unexpected ip for bare RequestCtx: %q. Expected 0.0.0.0", ip) - } - expectedLog := "#0012345700000000 - 0.0.0.0:0<->0.0.0.0:0 - GET http:/// - foo bar 10\n" - if logger.out != expectedLog { - t.Fatalf("Unexpected log output: %q. Expected %q", logger.out, expectedLog) - } -} - -func TestTimeoutHandlerSuccess(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - h := func(ctx *RequestCtx) { - if string(ctx.Path()) == "/" { - ctx.Success("aaa/bbb", []byte("real response")) - } - } - s := &Server{ - Handler: TimeoutHandler(h, 10*time.Second, "timeout!!!"), - } - serverCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - close(serverCh) - }() - - concurrency := 20 - clientCh := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - conn, err := ln.Dial() - if err != nil { - t.Errorf("unexepcted error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(conn) - verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") - clientCh <- struct{}{} - }() - } - - for i := 0; i < concurrency; i++ { - select { - case <-clientCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } -} - -func TestTimeoutHandlerTimeout(t *testing.T) { - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - readyCh := make(chan struct{}) - doneCh := make(chan struct{}) - h := func(ctx *RequestCtx) { - ctx.Success("aaa/bbb", []byte("real response")) - <-readyCh - doneCh <- struct{}{} - } - s := &Server{ - Handler: TimeoutHandler(h, 20*time.Millisecond, "timeout!!!"), - } - serverCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - close(serverCh) - }() - - concurrency := 20 - clientCh := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - conn, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(conn) - verifyResponse(t, br, StatusRequestTimeout, string(defaultContentType), "timeout!!!") - clientCh <- struct{}{} - }() - } - - for i := 0; i < concurrency; i++ { - select { - case <-clientCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - } - - close(readyCh) - for i := 0; i < concurrency; i++ { - select { - case <-doneCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case <-serverCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } -} - -func TestTimeoutHandlerTimeoutReuse(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - h := func(ctx *RequestCtx) { - if string(ctx.Path()) == "/timeout" { - time.Sleep(time.Second) - } - ctx.SetBodyString("ok") - } - s := &Server{ - Handler: TimeoutHandler(h, 500*time.Millisecond, "timeout!!!"), - } - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - br := bufio.NewReader(conn) - if _, err = conn.Write([]byte("GET /timeout HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Fatalf("unexpected error: %v", err) - } - verifyResponse(t, br, StatusRequestTimeout, string(defaultContentType), "timeout!!!") - - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Fatalf("unexpected error: %v", err) - } - verifyResponse(t, br, StatusOK, string(defaultContentType), "ok") - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestServerGetOnly(t *testing.T) { - t.Parallel() - - h := func(ctx *RequestCtx) { - if !ctx.IsGet() { - t.Errorf("non-get request: %q", ctx.Method()) - } - ctx.Success("foo/bar", []byte("success")) - } - s := &Server{ - Handler: h, - GetOnly: true, - } - - rw := &readWriter{} - rw.r.WriteString("POST /foo HTTP/1.1\r\nHost: google.com\r\nContent-Length: 5\r\nContent-Type: aaa\r\n\r\n12345") - - ch := make(chan error) - go func() { - ch <- s.ServeConn(rw) - }() - - select { - case err := <-ch: - if err == nil { - t.Fatal("expecting error") - } - if err != ErrGetOnly { - t.Fatalf("Unexpected error from serveConn: %v. Expecting %v", err, ErrGetOnly) - } - case <-time.After(100 * time.Millisecond): - t.Fatal("timeout") - } - - br := bufio.NewReader(&rw.w) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("unexpected error: %v", err) - } - statusCode := resp.StatusCode() - if statusCode != StatusBadRequest { - t.Fatalf("unexpected status code: %d. Expecting %d", statusCode, StatusBadRequest) - } - if !resp.ConnectionClose() { - t.Fatal("missing 'Connection: close' response header") - } -} - -func TestServerTimeoutErrorWithResponse(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - go func() { - ctx.Success("aaa/bbb", []byte("xxxyyy")) - }() - - var resp Response - - resp.SetStatusCode(123) - resp.SetBodyString("foobar. Should be ignored") - ctx.TimeoutErrorWithResponse(&resp) - - resp.SetStatusCode(456) - resp.ResetBody() - fmt.Fprintf(resp.BodyWriter(), "path=%s", ctx.Path()) - resp.Header.SetContentType("foo/bar") - ctx.TimeoutErrorWithResponse(&resp) - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo HTTP/1.1\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("GET /bar HTTP/1.1\r\nHost: google.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, 456, "foo/bar", "path=/foo") - verifyResponse(t, br, 456, "foo/bar", "path=/bar") - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("Unexpected error when reading remaining data: %v", err) - } - if len(data) != 0 { - t.Fatalf("Unexpected data read after the first response %q. Expecting %q", data, "") - } -} - -func TestServerTimeoutErrorWithCode(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - go func() { - ctx.Success("aaa/bbb", []byte("xxxyyy")) - }() - ctx.TimeoutErrorWithCode("should be ignored", 234) - ctx.TimeoutErrorWithCode("stolen ctx", StatusBadRequest) - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo HTTP/1.1\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("GET /foo HTTP/1.1\r\nHost: google.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, StatusBadRequest, string(defaultContentType), "stolen ctx") - verifyResponse(t, br, StatusBadRequest, string(defaultContentType), "stolen ctx") - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("Unexpected error when reading remaining data: %v", err) - } - if len(data) != 0 { - t.Fatalf("Unexpected data read after the first response %q. Expecting %q", data, "") - } -} - -func TestServerTimeoutError(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - go func() { - ctx.Success("aaa/bbb", []byte("xxxyyy")) - }() - ctx.TimeoutError("should be ignored") - ctx.TimeoutError("stolen ctx") - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo HTTP/1.1\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("GET /foo HTTP/1.1\r\nHost: google.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, StatusRequestTimeout, string(defaultContentType), "stolen ctx") - verifyResponse(t, br, StatusRequestTimeout, string(defaultContentType), "stolen ctx") - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("Unexpected error when reading remaining data: %v", err) - } - if len(data) != 0 { - t.Fatalf("Unexpected data read after the first response %q. Expecting %q", data, "") - } -} - -func TestServerMaxRequestsPerConn(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) {}, - MaxRequestsPerConn: 1, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo1 HTTP/1.1\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("GET /bar HTTP/1.1\r\nHost: aaa.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("Unexpected error when parsing response: %v", err) - } - if !resp.ConnectionClose() { - t.Fatal("Response must have 'connection: close' header") - } - verifyResponseHeader(t, &resp.Header, 200, 0, string(defaultContentType), "") - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("Unexpected error when reading remaining data: %v", err) - } - if len(data) != 0 { - t.Fatalf("Unexpected data read after the first response %q. Expecting %q", data, "") - } -} - -func TestServerConnectionClose(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.SetConnectionClose() - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo1 HTTP/1.1\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("GET /must/be/ignored HTTP/1.1\r\nHost: aaa.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - var resp Response - - if err := resp.Read(br); err != nil { - t.Fatalf("Unexpected error when parsing response: %v", err) - } - if !resp.ConnectionClose() { - t.Fatal("expecting Connection: close header") - } - - data, err := ioutil.ReadAll(br) - if err != nil { - t.Fatalf("Unexpected error when reading remaining data: %v", err) - } - if len(data) != 0 { - t.Fatalf("Unexpected data read after the first response %q. Expecting %q", data, "") - } -} - -func TestServerRequestNumAndTime(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - n := uint64(0) - var connT time.Time - s := &Server{ - Handler: func(ctx *RequestCtx) { - n++ - if ctx.ConnRequestNum() != n { - t.Errorf("unexpected request number: %d. Expecting %d", ctx.ConnRequestNum(), n) - } - if connT.IsZero() { - connT = ctx.ConnTime() - } - if ctx.ConnTime() != connT { - t.Errorf("unexpected serve conn time: %q. Expecting %q", ctx.ConnTime(), connT) - } - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo1 HTTP/1.1\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("GET /bar HTTP/1.1\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("GET /baz HTTP/1.1\r\nHost: google.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - if n != 3 { - t.Fatalf("unexpected number of requests served: %d. Expecting %d", n, 3) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, 200, string(defaultContentType), "") -} - -func TestServerEmptyResponse(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - // do nothing :) - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo1 HTTP/1.1\r\nHost: google.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, 200, string(defaultContentType), "") -} - -func TestServerLogger(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - // This test can't run parallel as it modifies globalConnID. - - cl := &testLogger{} - s := &Server{ - Handler: func(ctx *RequestCtx) { - logger := cl - h := &ctx.Request.Header - logger.Printf("begin") - ctx.Success("text/html", []byte(fmt.Sprintf("requestURI=%s, body=%q, remoteAddr=%s", - h.RequestURI(), ctx.Request.Body(), ctx.RemoteAddr()))) - logger.Printf("end") - }, - Logger: cl, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo1 HTTP/1.1\r\nHost: google.com\r\n\r\n") - rw.r.WriteString("POST /foo2 HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: 5\r\nContent-Type: aa\r\n\r\nabcde") - - rwx := &readWriterRemoteAddr{ - rw: rw, - addr: &net.TCPAddr{ - IP: []byte{1, 2, 3, 4}, - Port: 8765, - }, - } - - globalConnID = 0 - - if err := s.ServeConn(rwx); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, 200, "text/html", "requestURI=/foo1, body=\"\", remoteAddr=1.2.3.4:8765") - verifyResponse(t, br, 200, "text/html", "requestURI=/foo2, body=\"abcde\", remoteAddr=1.2.3.4:8765") - - expectedLogOut := `#0000000100000001 - 1.2.3.4:8765<->1.2.3.4:8765 - GET http://google.com/foo1 - begin -#0000000100000001 - 1.2.3.4:8765<->1.2.3.4:8765 - GET http://google.com/foo1 - end -#0000000100000002 - 1.2.3.4:8765<->1.2.3.4:8765 - POST http://aaa.com/foo2 - begin -#0000000100000002 - 1.2.3.4:8765<->1.2.3.4:8765 - POST http://aaa.com/foo2 - end -` - if cl.out != expectedLogOut { - t.Fatalf("Unexpected logger output: %q. Expected %q", cl.out, expectedLogOut) - } -} - -func TestServerRemoteAddr(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - h := &ctx.Request.Header - ctx.Success("text/html", []byte(fmt.Sprintf("requestURI=%s, remoteAddr=%s, remoteIP=%s", - h.RequestURI(), ctx.RemoteAddr(), ctx.RemoteIP()))) - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo1 HTTP/1.1\r\nHost: google.com\r\n\r\n") - - rwx := &readWriterRemoteAddr{ - rw: rw, - addr: &net.TCPAddr{ - IP: []byte{1, 2, 3, 4}, - Port: 8765, - }, - } - - if err := s.ServeConn(rwx); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, 200, "text/html", "requestURI=/foo1, remoteAddr=1.2.3.4:8765, remoteIP=1.2.3.4") -} - -func TestServerCustomRemoteAddr(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - customRemoteAddrHandler := func(h RequestHandler) RequestHandler { - return func(ctx *RequestCtx) { - ctx.SetRemoteAddr(&net.TCPAddr{ - IP: []byte{1, 2, 3, 5}, - Port: 0, - }) - h(ctx) - } - } - - s := &Server{ - Handler: customRemoteAddrHandler(func(ctx *RequestCtx) { - h := &ctx.Request.Header - ctx.Success("text/html", []byte(fmt.Sprintf("requestURI=%s, remoteAddr=%s, remoteIP=%s", - h.RequestURI(), ctx.RemoteAddr(), ctx.RemoteIP()))) - }), - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo1 HTTP/1.1\r\nHost: google.com\r\n\r\n") - - rwx := &readWriterRemoteAddr{ - rw: rw, - addr: &net.TCPAddr{ - IP: []byte{1, 2, 3, 4}, - Port: 8765, - }, - } - - if err := s.ServeConn(rwx); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, 200, "text/html", "requestURI=/foo1, remoteAddr=1.2.3.5:0, remoteIP=1.2.3.5") -} - -type readWriterRemoteAddr struct { - net.Conn - rw io.ReadWriteCloser - addr net.Addr -} - -func (rw *readWriterRemoteAddr) Close() error { - return rw.rw.Close() -} - -func (rw *readWriterRemoteAddr) Read(b []byte) (int, error) { - return rw.rw.Read(b) -} - -func (rw *readWriterRemoteAddr) Write(b []byte) (int, error) { - return rw.rw.Write(b) -} - -func (rw *readWriterRemoteAddr) RemoteAddr() net.Addr { - return rw.addr -} - -func (rw *readWriterRemoteAddr) LocalAddr() net.Addr { - return rw.addr -} - -func TestServerConnError(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Error("foobar", 423) - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo/bar?baz HTTP/1.1\r\nHost: google.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("Unexpected error when reading response: %v", err) - } - if resp.Header.StatusCode() != 423 { - t.Fatalf("Unexpected status code %d. Expected %d", resp.Header.StatusCode(), 423) - } - if resp.Header.ContentLength() != 6 { - t.Fatalf("Unexpected Content-Length %d. Expected %d", resp.Header.ContentLength(), 6) - } - if !bytes.Equal(resp.Header.Peek(HeaderContentType), defaultContentType) { - t.Fatalf("Unexpected Content-Type %q. Expected %q", resp.Header.Peek(HeaderContentType), defaultContentType) - } - if !bytes.Equal(resp.Body(), []byte("foobar")) { - t.Fatalf("Unexpected body %q. Expected %q", resp.Body(), "foobar") - } -} - -func TestServeConnSingleRequest(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - h := &ctx.Request.Header - ctx.Success("aaa", []byte(fmt.Sprintf("requestURI=%s, host=%s", h.RequestURI(), h.Peek(HeaderHost)))) - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo/bar?baz HTTP/1.1\r\nHost: google.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, 200, "aaa", "requestURI=/foo/bar?baz, host=google.com") -} - -func TestServeConnMultiRequests(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - h := &ctx.Request.Header - ctx.Success("aaa", []byte(fmt.Sprintf("requestURI=%s, host=%s", h.RequestURI(), h.Peek(HeaderHost)))) - }, - } - - rw := &readWriter{} - rw.r.WriteString("GET /foo/bar?baz HTTP/1.1\r\nHost: google.com\r\n\r\nGET /abc HTTP/1.1\r\nHost: foobar.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - verifyResponse(t, br, 200, "aaa", "requestURI=/foo/bar?baz, host=google.com") - verifyResponse(t, br, 200, "aaa", "requestURI=/abc, host=foobar.com") -} - -func TestShutdown(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - time.Sleep(time.Millisecond * 500) - ctx.Success("aaa/bbb", []byte("real response")) - }, - } - serveCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - _, err := ln.Dial() - if err == nil { - t.Error("server is still listening") - } - serveCh <- struct{}{} - }() - clientCh := make(chan struct{}) - go func() { - conn, err := ln.Dial() - if err != nil { - t.Errorf("unexepcted error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(conn) - resp := verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") - verifyResponseHeaderConnection(t, &resp.Header, "") - clientCh <- struct{}{} - }() - time.Sleep(time.Millisecond * 100) - shutdownCh := make(chan struct{}) - go func() { - if err := s.Shutdown(); err != nil { - t.Errorf("unexepcted error: %v", err) - } - shutdownCh <- struct{}{} - }() - done := 0 - for { - select { - case <-time.After(time.Second * 2): - t.Fatal("shutdown took too long") - case <-serveCh: - done++ - case <-clientCh: - done++ - case <-shutdownCh: - done++ - } - if done == 3 { - return - } - } -} - -func TestCloseOnShutdown(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - time.Sleep(time.Millisecond * 500) - ctx.Success("aaa/bbb", []byte("real response")) - }, - CloseOnShutdown: true, - } - serveCh := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - _, err := ln.Dial() - if err == nil { - t.Error("server is still listening") - } - serveCh <- struct{}{} - }() - clientCh := make(chan struct{}) - go func() { - conn, err := ln.Dial() - if err != nil { - t.Errorf("unexepcted error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(conn) - resp := verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") - verifyResponseHeaderConnection(t, &resp.Header, "close") - clientCh <- struct{}{} - }() - time.Sleep(time.Millisecond * 100) - shutdownCh := make(chan struct{}) - go func() { - if err := s.Shutdown(); err != nil { - t.Errorf("unexepcted error: %v", err) - } - shutdownCh <- struct{}{} - }() - done := 0 - for { - select { - case <-time.After(time.Second): - t.Fatal("shutdown took too long") - case <-serveCh: - done++ - case <-clientCh: - done++ - case <-shutdownCh: - done++ - } - if done == 3 { - return - } - } -} - -func TestShutdownReuse(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Success("aaa/bbb", []byte("real response")) - }, - ReadTimeout: time.Millisecond * 100, - Logger: &testLogger{}, // Ignore log output. - } - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexepcted error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Fatalf("unexpected error: %v", err) - } - br := bufio.NewReader(conn) - verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") - if err := s.Shutdown(); err != nil { - t.Fatalf("unexepcted error: %v", err) - } - ln = fasthttputil.NewInmemoryListener() - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - conn, err = ln.Dial() - if err != nil { - t.Fatalf("unexepcted error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Fatalf("unexpected error: %v", err) - } - br = bufio.NewReader(conn) - verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") - if err := s.Shutdown(); err != nil { - t.Fatalf("unexepcted error: %v", err) - } -} - -func TestShutdownDone(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - <-ctx.Done() - ctx.Success("aaa/bbb", []byte("real response")) - }, - } - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexepcted error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Fatalf("unexpected error: %v", err) - } - go func() { - // Shutdown won't return if the connection doesn't close, - // which doesn't happen until we read the response. - if err := s.Shutdown(); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - // We can only reach this point and get a valid response - // if reading from ctx.Done() returned. - br := bufio.NewReader(conn) - verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") -} - -func TestShutdownErr(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - // This will panic, but I was not able to intercept with recover() - c, cancel := context.WithCancel(ctx) - defer cancel() - <-c.Done() - ctx.Success("aaa/bbb", []byte("real response")) - }, - } - - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexepcted error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Fatalf("unexpected error: %v", err) - } - go func() { - // Shutdown won't return if the connection doesn't close, - // which doesn't happen until we read the response. - if err := s.Shutdown(); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - // We can only reach this point and get a valid response - // if reading from ctx.Done() returned. - br := bufio.NewReader(conn) - verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") -} - -func TestShutdownCloseIdleConns(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Success("aaa/bbb", []byte("real response")) - }, - } - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexepcted error: %v", err) - } - - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Errorf("unexpected error: %v", err) - } - br := bufio.NewReader(conn) - verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") - - shutdownErr := make(chan error) - go func() { - shutdownErr <- s.Shutdown() - }() - - timer := time.NewTimer(time.Second) - select { - case <-timer.C: - t.Fatal("idle connections not closed on shutdown") - case err = <-shutdownErr: - if err != nil { - t.Errorf("unexepcted error: %v", err) - } - } -} - -func TestMultipleServe(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Success("aaa/bbb", []byte("real response")) - }, - } - - ln1 := fasthttputil.NewInmemoryListener() - ln2 := fasthttputil.NewInmemoryListener() - - go func() { - if err := s.Serve(ln1); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - go func() { - if err := s.Serve(ln2); err != nil { - t.Errorf("unexepcted error: %v", err) - } - }() - - conn, err := ln1.Dial() - if err != nil { - t.Fatalf("unexepcted error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Fatalf("unexpected error: %v", err) - } - br := bufio.NewReader(conn) - verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") - - conn, err = ln2.Dial() - if err != nil { - t.Fatalf("unexepcted error: %v", err) - } - if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { - t.Fatalf("unexpected error: %v", err) - } - br = bufio.NewReader(conn) - verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") -} - -func TestMaxBodySizePerRequest(t *testing.T) { - t.Parallel() - - s := &Server{ - Handler: func(ctx *RequestCtx) { - // do nothing :) - }, - HeaderReceived: func(header *RequestHeader) RequestConfig { - return RequestConfig{ - MaxRequestBodySize: 5 << 10, - } - }, - ReadTimeout: time.Second * 5, - WriteTimeout: time.Second * 5, - MaxRequestBodySize: 1 << 20, - } - - rw := &readWriter{} - rw.r.WriteString(fmt.Sprintf("POST /foo2 HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: %d\r\nContent-Type: aa\r\n\r\n%s", (5<<10)+1, strings.Repeat("a", (5<<10)+1))) - - if err := s.ServeConn(rw); err != ErrBodyTooLarge { - t.Fatalf("Unexpected error from serveConn: %v", err) - } -} - -func TestStreamRequestBody(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - part1 := strings.Repeat("1", 1<<15) - part2 := strings.Repeat("2", 1<<16) - contentLength := len(part1) + len(part2) - next := make(chan struct{}) - - s := &Server{ - Handler: func(ctx *RequestCtx) { - checkReader(t, ctx.RequestBodyStream(), part1) - close(next) - checkReader(t, ctx.RequestBodyStream(), part2) - }, - StreamRequestBody: true, - Logger: &testLogger{}, - } - - pipe := fasthttputil.NewPipeConns() - cc, sc := pipe.Conn1(), pipe.Conn2() - //write headers and part1 body - if _, err := cc.Write([]byte(fmt.Sprintf("POST /foo2 HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: %d\r\nContent-Type: aa\r\n\r\n", contentLength))); err != nil { - t.Fatal(err) - } - if _, err := cc.Write([]byte(part1)); err != nil { - t.Fatal(err) - } - - ch := make(chan error) - go func() { - ch <- s.ServeConn(sc) - }() - - select { - case <-next: - case <-time.After(500 * time.Millisecond): - t.Fatal("part1 timeout") - } - - if _, err := cc.Write([]byte(part2)); err != nil { - t.Fatal(err) - } - if err := sc.Close(); err != nil { - t.Fatal(err) - } - - select { - case err := <-ch: - if err != nil && err.Error() != "connection closed" { // fasthttputil.errConnectionClosed is private so do a string match. - t.Fatalf("Unexpected error from serveConn: %v", err) - } - case <-time.After(500 * time.Millisecond): - t.Fatal("part2 timeout") - } -} - -func TestStreamRequestBodyExceedMaxSize(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - part1 := strings.Repeat("1", 1<<18) - part2 := strings.Repeat("2", 1<<20-1<<18) - contentLength := len(part1) + len(part2) - next := make(chan struct{}) - - s := &Server{ - Handler: func(ctx *RequestCtx) { - checkReader(t, ctx.RequestBodyStream(), part1) - close(next) - checkReader(t, ctx.RequestBodyStream(), part2) - }, - DisableKeepalive: true, - StreamRequestBody: true, - MaxRequestBodySize: 1, - } - - pipe := fasthttputil.NewPipeConns() - cc, sc := pipe.Conn1(), pipe.Conn2() - //write headers and part1 body - if _, err := cc.Write([]byte(fmt.Sprintf("POST /foo2 HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: %d\r\nContent-Type: aa\r\n\r\n%s", contentLength, part1))); err != nil { - t.Error(err) - } - - ch := make(chan error) - go func() { - ch <- s.ServeConn(sc) - }() - - select { - case <-next: - case <-time.After(500 * time.Millisecond): - t.Fatal("part1 timeout") - } - - if _, err := cc.Write([]byte(part2)); err != nil { - t.Error(err) - } - - select { - case err := <-ch: - if err != nil { - t.Error(err) - } - case <-time.After(500 * time.Millisecond): - t.Fatal("part2 timeout") - } -} - -func TestStreamBodyRequestContentLength(t *testing.T) { - t.Parallel() - content := strings.Repeat("1", 1<<15) // 32K - contentLength := len(content) - - s := &Server{ - Handler: func(ctx *RequestCtx) { - realContentLength := ctx.Request.Header.ContentLength() - if realContentLength != contentLength { - t.Fatal("incorrect content length") - } - }, - MaxRequestBodySize: 1 * 1024 * 1024, // 1M - StreamRequestBody: true, - } - - pipe := fasthttputil.NewPipeConns() - cc, sc := pipe.Conn1(), pipe.Conn2() - if _, err := cc.Write([]byte(fmt.Sprintf("POST /foo2 HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: %d\r\nContent-Type: aa\r\n\r\n%s", contentLength, content))); err != nil { - t.Fatal(err) - } - - ch := make(chan error) - go func() { - ch <- s.ServeConn(sc) - }() - - if err := sc.Close(); err != nil { - t.Fatal(err) - } - - select { - case err := <-ch: - if err == nil || err.Error() != "connection closed" { // fasthttputil.errConnectionClosed is private so do a string match. - t.Fatalf("Unexpected error from serveConn: %v", err) - } - case <-time.After(time.Second): - t.Fatal("test timeout") - } -} - -func checkReader(t *testing.T, r io.Reader, expected string) { - b := make([]byte, len(expected)) - if _, err := io.ReadFull(r, b); err != nil { - t.Fatalf("Unexpected error from reader: %v", err) - } - if string(b) != expected { - t.Fatal("incorrect request body") - } -} - -func TestMaxReadTimeoutPerRequest(t *testing.T) { - t.Parallel() - - headers := []byte(fmt.Sprintf("POST /foo2 HTTP/1.1\r\nHost: aaa.com\r\nContent-Length: %d\r\nContent-Type: aa\r\n\r\n", 5*1024)) - s := &Server{ - Handler: func(_ *RequestCtx) { - t.Error("shouldn't reach handler") - }, - HeaderReceived: func(header *RequestHeader) RequestConfig { - return RequestConfig{ - ReadTimeout: time.Millisecond, - } - }, - ReadBufferSize: len(headers), - ReadTimeout: time.Second * 5, - WriteTimeout: time.Second * 5, - } - - pipe := fasthttputil.NewPipeConns() - cc, sc := pipe.Conn1(), pipe.Conn2() - go func() { - //write headers - _, err := cc.Write(headers) - if err != nil { - t.Error(err) - } - //write body - for i := 0; i < 5*1024; i++ { - time.Sleep(time.Millisecond) - cc.Write([]byte{'a'}) //nolint:errcheck - } - }() - ch := make(chan error) - go func() { - ch <- s.ServeConn(sc) - }() - - select { - case err := <-ch: - if err == nil || err != nil && !strings.EqualFold(err.Error(), "timeout") { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - case <-time.After(time.Second): - t.Fatal("test timeout") - } -} - -func TestMaxWriteTimeoutPerRequest(t *testing.T) { - t.Parallel() - - headers := []byte("GET /foo2 HTTP/1.1\r\nHost: aaa.com\r\nContent-Type: aa\r\n\r\n") - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.SetBodyStreamWriter(func(w *bufio.Writer) { - var buf [192]byte - for { - w.Write(buf[:]) //nolint:errcheck - } - }) - }, - HeaderReceived: func(header *RequestHeader) RequestConfig { - return RequestConfig{ - WriteTimeout: time.Millisecond, - } - }, - ReadBufferSize: 192, - ReadTimeout: time.Second * 5, - WriteTimeout: time.Second * 5, - } - - pipe := fasthttputil.NewPipeConns() - cc, sc := pipe.Conn1(), pipe.Conn2() - - var resp Response - go func() { - //write headers - _, err := cc.Write(headers) - if err != nil { - t.Error(err) - } - br := bufio.NewReaderSize(cc, 192) - err = resp.Header.Read(br) - if err != nil { - t.Error(err) - } - - var chunk [192]byte - for { - time.Sleep(time.Millisecond) - br.Read(chunk[:]) //nolint:errcheck - } - }() - ch := make(chan error) - go func() { - ch <- s.ServeConn(sc) - }() - - select { - case err := <-ch: - if err == nil || err != nil && !strings.EqualFold(err.Error(), "timeout") { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - case <-time.After(time.Second): - t.Fatal("test timeout") - } -} - -func TestIncompleteBodyReturnsUnexpectedEOF(t *testing.T) { - t.Parallel() - - rw := &readWriter{} - rw.r.WriteString("POST /foo HTTP/1.1\r\nHost: google.com\r\nContent-Length: 5\r\n\r\n123") - s := &Server{ - Handler: func(ctx *RequestCtx) {}, - } - ch := make(chan error) - go func() { - ch <- s.ServeConn(rw) - }() - if err := <-ch; err == nil || err.Error() != "unexpected EOF" { - t.Fatal(err) - } -} - -func TestServerChunkedResponse(t *testing.T) { - t.Parallel() - - trailer := map[string]string{ - "AtEnd1": "1111", - "AtEnd2": "2222", - "AtEnd3": "3333", - } - - h := func(ctx *RequestCtx) { - ctx.Response.Header.DisableNormalizing() - ctx.Response.Header.Set("Transfer-Encoding", "chunked") - for k := range trailer { - err := ctx.Response.Header.AddTrailer(k) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - } - ctx.Response.SetBodyStreamWriter(func(w *bufio.Writer) { - for i := 0; i < 3; i++ { - fmt.Fprintf(w, "message %d", i) - if err := w.Flush(); err != nil { - t.Errorf("unexpected error: %v", err) - } - time.Sleep(time.Millisecond * 100) - } - }) - for k, v := range trailer { - ctx.Response.Header.Set(k, v) - } - } - s := &Server{ - Handler: h, - } - - rw := &readWriter{} - rw.r.WriteString("GET / HTTP/1.1\r\nHost: test.com\r\n\r\n") - - if err := s.ServeConn(rw); err != nil { - t.Fatalf("Unexpected error from serveConn: %v", err) - } - - br := bufio.NewReader(&rw.w) - var resp Response - if err := resp.Read(br); err != nil { - t.Fatalf("Unexpected error when reading response: %v", err) - } - if resp.Header.ContentLength() != -1 { - t.Fatalf("Unexpected Content-Length %d. Expected %d", resp.Header.ContentLength(), -1) - } - if !bytes.Equal(resp.Body(), []byte("message 0"+"message 1"+"message 2")) { - t.Fatalf("Unexpected body %q. Expected %q", resp.Body(), "foobar") - } - for k, v := range trailer { - h := resp.Header.Peek(k) - if !bytes.Equal(resp.Header.Peek(k), []byte(v)) { - t.Fatalf("Unexpected trailer %q. Expected %q. Got %q", k, v, h) - } - } -} - -func verifyResponse(t *testing.T, r *bufio.Reader, expectedStatusCode int, expectedContentType, expectedBody string) *Response { - var resp Response - if err := resp.Read(r); err != nil { - t.Fatalf("Unexpected error when parsing response: %v", err) - } - - if !bytes.Equal(resp.Body(), []byte(expectedBody)) { - t.Fatalf("Unexpected body %q. Expected %q", resp.Body(), []byte(expectedBody)) - } - verifyResponseHeader(t, &resp.Header, expectedStatusCode, len(resp.Body()), expectedContentType, "") - return &resp -} - -type readWriter struct { - net.Conn - r bytes.Buffer - w bytes.Buffer -} - -func (rw *readWriter) Close() error { - return nil -} - -func (rw *readWriter) Read(b []byte) (int, error) { - return rw.r.Read(b) -} - -func (rw *readWriter) Write(b []byte) (int, error) { - return rw.w.Write(b) -} - -func (rw *readWriter) RemoteAddr() net.Addr { - return zeroTCPAddr -} - -func (rw *readWriter) LocalAddr() net.Addr { - return zeroTCPAddr -} - -func (rw *readWriter) SetDeadline(t time.Time) error { - return nil -} - -func (rw *readWriter) SetReadDeadline(t time.Time) error { - return nil -} - -func (rw *readWriter) SetWriteDeadline(t time.Time) error { - return nil -} - -type testLogger struct { - lock sync.Mutex - out string -} - -func (cl *testLogger) Printf(format string, args ...interface{}) { - cl.lock.Lock() - cl.out += fmt.Sprintf(format, args...)[6:] + "\n" - cl.lock.Unlock() -} diff --git a/lib/fasthttp/server_timing_test.go b/lib/fasthttp/server_timing_test.go deleted file mode 100644 index 0f2d4a7db..000000000 --- a/lib/fasthttp/server_timing_test.go +++ /dev/null @@ -1,461 +0,0 @@ -package fasthttp - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - "runtime" - "sync" - "sync/atomic" - "testing" - "time" -) - -var defaultClientsCount = runtime.NumCPU() - -func BenchmarkRequestCtxRedirect(b *testing.B) { - b.RunParallel(func(pb *testing.PB) { - var ctx RequestCtx - for pb.Next() { - ctx.Request.SetRequestURI("http://aaa.com/fff/ss.html?sdf") - ctx.Redirect("/foo/bar?baz=111", StatusFound) - } - }) -} - -func BenchmarkServerGet1ReqPerConn(b *testing.B) { - benchmarkServerGet(b, defaultClientsCount, 1) -} - -func BenchmarkServerGet2ReqPerConn(b *testing.B) { - benchmarkServerGet(b, defaultClientsCount, 2) -} - -func BenchmarkServerGet10ReqPerConn(b *testing.B) { - benchmarkServerGet(b, defaultClientsCount, 10) -} - -func BenchmarkServerGet10KReqPerConn(b *testing.B) { - benchmarkServerGet(b, defaultClientsCount, 10000) -} - -func BenchmarkNetHTTPServerGet1ReqPerConn(b *testing.B) { - benchmarkNetHTTPServerGet(b, defaultClientsCount, 1) -} - -func BenchmarkNetHTTPServerGet2ReqPerConn(b *testing.B) { - benchmarkNetHTTPServerGet(b, defaultClientsCount, 2) -} - -func BenchmarkNetHTTPServerGet10ReqPerConn(b *testing.B) { - benchmarkNetHTTPServerGet(b, defaultClientsCount, 10) -} - -func BenchmarkNetHTTPServerGet10KReqPerConn(b *testing.B) { - benchmarkNetHTTPServerGet(b, defaultClientsCount, 10000) -} - -func BenchmarkServerPost1ReqPerConn(b *testing.B) { - benchmarkServerPost(b, defaultClientsCount, 1) -} - -func BenchmarkServerPost2ReqPerConn(b *testing.B) { - benchmarkServerPost(b, defaultClientsCount, 2) -} - -func BenchmarkServerPost10ReqPerConn(b *testing.B) { - benchmarkServerPost(b, defaultClientsCount, 10) -} - -func BenchmarkServerPost10KReqPerConn(b *testing.B) { - benchmarkServerPost(b, defaultClientsCount, 10000) -} - -func BenchmarkNetHTTPServerPost1ReqPerConn(b *testing.B) { - benchmarkNetHTTPServerPost(b, defaultClientsCount, 1) -} - -func BenchmarkNetHTTPServerPost2ReqPerConn(b *testing.B) { - benchmarkNetHTTPServerPost(b, defaultClientsCount, 2) -} - -func BenchmarkNetHTTPServerPost10ReqPerConn(b *testing.B) { - benchmarkNetHTTPServerPost(b, defaultClientsCount, 10) -} - -func BenchmarkNetHTTPServerPost10KReqPerConn(b *testing.B) { - benchmarkNetHTTPServerPost(b, defaultClientsCount, 10000) -} - -func BenchmarkServerGet1ReqPerConn10KClients(b *testing.B) { - benchmarkServerGet(b, 10000, 1) -} - -func BenchmarkServerGet2ReqPerConn10KClients(b *testing.B) { - benchmarkServerGet(b, 10000, 2) -} - -func BenchmarkServerGet10ReqPerConn10KClients(b *testing.B) { - benchmarkServerGet(b, 10000, 10) -} - -func BenchmarkServerGet100ReqPerConn10KClients(b *testing.B) { - benchmarkServerGet(b, 10000, 100) -} - -func BenchmarkNetHTTPServerGet1ReqPerConn10KClients(b *testing.B) { - benchmarkNetHTTPServerGet(b, 10000, 1) -} - -func BenchmarkNetHTTPServerGet2ReqPerConn10KClients(b *testing.B) { - benchmarkNetHTTPServerGet(b, 10000, 2) -} - -func BenchmarkNetHTTPServerGet10ReqPerConn10KClients(b *testing.B) { - benchmarkNetHTTPServerGet(b, 10000, 10) -} - -func BenchmarkNetHTTPServerGet100ReqPerConn10KClients(b *testing.B) { - benchmarkNetHTTPServerGet(b, 10000, 100) -} - -func BenchmarkServerHijack(b *testing.B) { - clientsCount := 1000 - requestsPerConn := 10000 - ch := make(chan struct{}, b.N) - responseBody := []byte("123") - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Hijack(func(c net.Conn) { - // emulate server loop :) - err := ServeConn(c, func(ctx *RequestCtx) { - ctx.Success("foobar", responseBody) - registerServedRequest(b, ch) - }) - if err != nil { - b.Fatalf("error when serving connection") - } - }) - ctx.Success("foobar", responseBody) - registerServedRequest(b, ch) - }, - Concurrency: 16 * clientsCount, - } - req := "GET /foo HTTP/1.1\r\nHost: google.com\r\n\r\n" - benchmarkServer(b, s, clientsCount, requestsPerConn, req) - verifyRequestsServed(b, ch) -} - -func BenchmarkServerMaxConnsPerIP(b *testing.B) { - clientsCount := 1000 - requestsPerConn := 10 - ch := make(chan struct{}, b.N) - responseBody := []byte("123") - s := &Server{ - Handler: func(ctx *RequestCtx) { - ctx.Success("foobar", responseBody) - registerServedRequest(b, ch) - }, - MaxConnsPerIP: clientsCount * 2, - Concurrency: 16 * clientsCount, - } - req := "GET /foo HTTP/1.1\r\nHost: google.com\r\n\r\n" - benchmarkServer(b, s, clientsCount, requestsPerConn, req) - verifyRequestsServed(b, ch) -} - -func BenchmarkServerTimeoutError(b *testing.B) { - clientsCount := 10 - requestsPerConn := 1 - ch := make(chan struct{}, b.N) - n := uint32(0) - responseBody := []byte("123") - s := &Server{ - Handler: func(ctx *RequestCtx) { - if atomic.AddUint32(&n, 1)&7 == 0 { - ctx.TimeoutError("xxx") - go func() { - ctx.Success("foobar", responseBody) - }() - } else { - ctx.Success("foobar", responseBody) - } - registerServedRequest(b, ch) - }, - Concurrency: 16 * clientsCount, - } - req := "GET /foo HTTP/1.1\r\nHost: google.com\r\n\r\n" - benchmarkServer(b, s, clientsCount, requestsPerConn, req) - verifyRequestsServed(b, ch) -} - -type fakeServerConn struct { - net.TCPConn - ln *fakeListener - requestsCount int - pos int - closed uint32 -} - -func (c *fakeServerConn) Read(b []byte) (int, error) { - nn := 0 - reqLen := len(c.ln.request) - for len(b) > 0 { - if c.requestsCount == 0 { - if nn == 0 { - return 0, io.EOF - } - return nn, nil - } - pos := c.pos % reqLen - n := copy(b, c.ln.request[pos:]) - b = b[n:] - nn += n - c.pos += n - if n+pos == reqLen { - c.requestsCount-- - } - } - return nn, nil -} - -func (c *fakeServerConn) Write(b []byte) (int, error) { - return len(b), nil -} - -var fakeAddr = net.TCPAddr{ - IP: []byte{1, 2, 3, 4}, - Port: 12345, -} - -func (c *fakeServerConn) RemoteAddr() net.Addr { - return &fakeAddr -} - -func (c *fakeServerConn) Close() error { - if atomic.AddUint32(&c.closed, 1) == 1 { - c.ln.ch <- c - } - return nil -} - -func (c *fakeServerConn) SetReadDeadline(t time.Time) error { - return nil -} - -func (c *fakeServerConn) SetWriteDeadline(t time.Time) error { - return nil -} - -type fakeListener struct { - lock sync.Mutex - requestsCount int - requestsPerConn int - request []byte - ch chan *fakeServerConn - done chan struct{} - closed bool -} - -func (ln *fakeListener) Accept() (net.Conn, error) { - ln.lock.Lock() - if ln.requestsCount == 0 { - ln.lock.Unlock() - for len(ln.ch) < cap(ln.ch) { - time.Sleep(10 * time.Millisecond) - } - ln.lock.Lock() - if !ln.closed { - close(ln.done) - ln.closed = true - } - ln.lock.Unlock() - return nil, io.EOF - } - requestsCount := ln.requestsPerConn - if requestsCount > ln.requestsCount { - requestsCount = ln.requestsCount - } - ln.requestsCount -= requestsCount - ln.lock.Unlock() - - c := <-ln.ch - c.requestsCount = requestsCount - c.closed = 0 - c.pos = 0 - - return c, nil -} - -func (ln *fakeListener) Close() error { - return nil -} - -func (ln *fakeListener) Addr() net.Addr { - return &fakeAddr -} - -func newFakeListener(requestsCount, clientsCount, requestsPerConn int, request string) *fakeListener { - ln := &fakeListener{ - requestsCount: requestsCount, - requestsPerConn: requestsPerConn, - request: []byte(request), - ch: make(chan *fakeServerConn, clientsCount), - done: make(chan struct{}), - } - for i := 0; i < clientsCount; i++ { - ln.ch <- &fakeServerConn{ - ln: ln, - } - } - return ln -} - -var ( - fakeResponse = []byte("Hello, world!") - getRequest = "GET /foobar?baz HTTP/1.1\r\nHost: google.com\r\nUser-Agent: aaa/bbb/ccc/ddd/eee Firefox Chrome MSIE Opera\r\n" + - "Referer: http://example.com/aaa?bbb=ccc\r\nCookie: foo=bar; baz=baraz; aa=aakslsdweriwereowriewroire\r\n\r\n" - postRequest = fmt.Sprintf("POST /foobar?baz HTTP/1.1\r\nHost: google.com\r\nContent-Type: foo/bar\r\nContent-Length: %d\r\n"+ - "User-Agent: Opera Chrome MSIE Firefox and other/1.2.34\r\nReferer: http://google.com/aaaa/bbb/ccc\r\n"+ - "Cookie: foo=bar; baz=baraz; aa=aakslsdweriwereowriewroire\r\n\r\n%q", - len(fakeResponse), fakeResponse) -) - -func benchmarkServerGet(b *testing.B, clientsCount, requestsPerConn int) { - ch := make(chan struct{}, b.N) - s := &Server{ - Handler: func(ctx *RequestCtx) { - if !ctx.IsGet() { - b.Fatalf("Unexpected request method: %q", ctx.Method()) - } - ctx.Success("text/plain", fakeResponse) - if requestsPerConn == 1 { - ctx.SetConnectionClose() - } - registerServedRequest(b, ch) - }, - Concurrency: 16 * clientsCount, - } - benchmarkServer(b, s, clientsCount, requestsPerConn, getRequest) - verifyRequestsServed(b, ch) -} - -func benchmarkNetHTTPServerGet(b *testing.B, clientsCount, requestsPerConn int) { - ch := make(chan struct{}, b.N) - s := &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.Method != MethodGet { - b.Fatalf("Unexpected request method: %q", req.Method) - } - h := w.Header() - h.Set("Content-Type", "text/plain") - if requestsPerConn == 1 { - h.Set(HeaderConnection, "close") - } - w.Write(fakeResponse) //nolint:errcheck - registerServedRequest(b, ch) - }), - } - benchmarkServer(b, s, clientsCount, requestsPerConn, getRequest) - verifyRequestsServed(b, ch) -} - -func benchmarkServerPost(b *testing.B, clientsCount, requestsPerConn int) { - ch := make(chan struct{}, b.N) - s := &Server{ - Handler: func(ctx *RequestCtx) { - if !ctx.IsPost() { - b.Fatalf("Unexpected request method: %q", ctx.Method()) - } - body := ctx.Request.Body() - if !bytes.Equal(body, fakeResponse) { - b.Fatalf("Unexpected body %q. Expected %q", body, fakeResponse) - } - ctx.Success("text/plain", body) - if requestsPerConn == 1 { - ctx.SetConnectionClose() - } - registerServedRequest(b, ch) - }, - Concurrency: 16 * clientsCount, - } - benchmarkServer(b, s, clientsCount, requestsPerConn, postRequest) - verifyRequestsServed(b, ch) -} - -func benchmarkNetHTTPServerPost(b *testing.B, clientsCount, requestsPerConn int) { - ch := make(chan struct{}, b.N) - s := &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.Method != MethodPost { - b.Fatalf("Unexpected request method: %q", req.Method) - } - body, err := ioutil.ReadAll(req.Body) - if err != nil { - b.Fatalf("Unexpected error: %v", err) - } - req.Body.Close() - if !bytes.Equal(body, fakeResponse) { - b.Fatalf("Unexpected body %q. Expected %q", body, fakeResponse) - } - h := w.Header() - h.Set("Content-Type", "text/plain") - if requestsPerConn == 1 { - h.Set(HeaderConnection, "close") - } - w.Write(body) //nolint:errcheck - registerServedRequest(b, ch) - }), - } - benchmarkServer(b, s, clientsCount, requestsPerConn, postRequest) - verifyRequestsServed(b, ch) -} - -func registerServedRequest(b *testing.B, ch chan<- struct{}) { - select { - case ch <- struct{}{}: - default: - b.Fatalf("More than %d requests served", cap(ch)) - } -} - -func verifyRequestsServed(b *testing.B, ch <-chan struct{}) { - requestsServed := 0 - for len(ch) > 0 { - <-ch - requestsServed++ - } - requestsSent := b.N - for requestsServed < requestsSent { - select { - case <-ch: - requestsServed++ - case <-time.After(100 * time.Millisecond): - b.Fatalf("Unexpected number of requests served %d. Expected %d", requestsServed, requestsSent) - } - } -} - -type realServer interface { - Serve(ln net.Listener) error -} - -func benchmarkServer(b *testing.B, s realServer, clientsCount, requestsPerConn int, request string) { - ln := newFakeListener(b.N, clientsCount, requestsPerConn, request) - ch := make(chan struct{}) - go func() { - s.Serve(ln) //nolint:errcheck - ch <- struct{}{} - }() - - <-ln.done - - select { - case <-ch: - case <-time.After(10 * time.Second): - b.Fatalf("Server.Serve() didn't stop") - } -} diff --git a/lib/fasthttp/stackless/func_test.go b/lib/fasthttp/stackless/func_test.go deleted file mode 100644 index 6b2a8d5f2..000000000 --- a/lib/fasthttp/stackless/func_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package stackless - -import ( - "fmt" - "sync/atomic" - "testing" - "time" -) - -func TestNewFuncSimple(t *testing.T) { - t.Parallel() - - var n uint64 - f := NewFunc(func(ctx interface{}) { - atomic.AddUint64(&n, uint64(ctx.(int))) - }) - - iterations := 4 * 1024 - for i := 0; i < iterations; i++ { - if !f(2) { - t.Fatalf("f mustn't return false") - } - } - if n != uint64(2*iterations) { - t.Fatalf("Unexpected n: %d. Expecting %d", n, 2*iterations) - } -} - -func TestNewFuncMulti(t *testing.T) { - t.Parallel() - - var n1, n2 uint64 - f1 := NewFunc(func(ctx interface{}) { - atomic.AddUint64(&n1, uint64(ctx.(int))) - }) - f2 := NewFunc(func(ctx interface{}) { - atomic.AddUint64(&n2, uint64(ctx.(int))) - }) - - iterations := 4 * 1024 - - f1Done := make(chan error, 1) - go func() { - var err error - for i := 0; i < iterations; i++ { - if !f1(3) { - err = fmt.Errorf("f1 mustn't return false") - break - } - } - f1Done <- err - }() - - f2Done := make(chan error, 1) - go func() { - var err error - for i := 0; i < iterations; i++ { - if !f2(5) { - err = fmt.Errorf("f2 mustn't return false") - break - } - } - f2Done <- err - }() - - select { - case err := <-f1Done: - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - case <-time.After(time.Second): - t.Fatalf("timeout") - } - - select { - case err := <-f2Done: - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - case <-time.After(time.Second): - t.Fatalf("timeout") - } - - if n1 != uint64(3*iterations) { - t.Fatalf("unexpected n1: %d. Expecting %d", n1, 3*iterations) - } - if n2 != uint64(5*iterations) { - t.Fatalf("unexpected n2: %d. Expecting %d", n2, 5*iterations) - } -} diff --git a/lib/fasthttp/stackless/func_timing_test.go b/lib/fasthttp/stackless/func_timing_test.go deleted file mode 100644 index cd4e4634d..000000000 --- a/lib/fasthttp/stackless/func_timing_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package stackless - -import ( - "sync/atomic" - "testing" -) - -func BenchmarkFuncOverhead(b *testing.B) { - var n uint64 - f := NewFunc(func(ctx interface{}) { - atomic.AddUint64(&n, *(ctx.(*uint64))) - }) - b.RunParallel(func(pb *testing.PB) { - x := uint64(1) - for pb.Next() { - if !f(&x) { - b.Fatalf("f mustn't return false") - } - } - }) - if n != uint64(b.N) { - b.Fatalf("unexected n: %d. Expecting %d", n, b.N) - } -} - -func BenchmarkFuncPure(b *testing.B) { - var n uint64 - f := func(x *uint64) { - atomic.AddUint64(&n, *x) - } - b.RunParallel(func(pb *testing.PB) { - x := uint64(1) - for pb.Next() { - f(&x) - } - }) - if n != uint64(b.N) { - b.Fatalf("unexected n: %d. Expecting %d", n, b.N) - } -} diff --git a/lib/fasthttp/stackless/writer_test.go b/lib/fasthttp/stackless/writer_test.go deleted file mode 100644 index 7f57bbdef..000000000 --- a/lib/fasthttp/stackless/writer_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package stackless - -import ( - "bytes" - "compress/flate" - "compress/gzip" - "fmt" - "io" - "io/ioutil" - "testing" - "time" -) - -func TestCompressFlateSerial(t *testing.T) { - t.Parallel() - - if err := testCompressFlate(); err != nil { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestCompressFlateConcurrent(t *testing.T) { - t.Parallel() - - if err := testConcurrent(testCompressFlate, 10); err != nil { - t.Fatalf("unexpected error: %v", err) - } -} - -func testCompressFlate() error { - return testWriter(func(w io.Writer) Writer { - zw, err := flate.NewWriter(w, flate.DefaultCompression) - if err != nil { - panic(fmt.Sprintf("BUG: unexpected error: %v", err)) - } - return zw - }, func(r io.Reader) io.Reader { - return flate.NewReader(r) - }) -} - -func TestCompressGzipSerial(t *testing.T) { - t.Parallel() - - if err := testCompressGzip(); err != nil { - t.Fatalf("unexpected error: %v", err) - } -} - -func TestCompressGzipConcurrent(t *testing.T) { - t.Parallel() - - if err := testConcurrent(testCompressGzip, 10); err != nil { - t.Fatalf("unexpected error: %v", err) - } -} - -func testCompressGzip() error { - return testWriter(func(w io.Writer) Writer { - return gzip.NewWriter(w) - }, func(r io.Reader) io.Reader { - zr, err := gzip.NewReader(r) - if err != nil { - panic(fmt.Sprintf("BUG: cannot create gzip reader: %v", err)) - } - return zr - }) -} - -func testWriter(newWriter NewWriterFunc, newReader func(io.Reader) io.Reader) error { - dstW := &bytes.Buffer{} - w := NewWriter(dstW, newWriter) - - for i := 0; i < 5; i++ { - if err := testWriterReuse(w, dstW, newReader); err != nil { - return fmt.Errorf("unexpected error when re-using writer on iteration %d: %w", i, err) - } - dstW = &bytes.Buffer{} - w.Reset(dstW) - } - - return nil -} - -func testWriterReuse(w Writer, r io.Reader, newReader func(io.Reader) io.Reader) error { - wantW := &bytes.Buffer{} - mw := io.MultiWriter(w, wantW) - for i := 0; i < 30; i++ { - fmt.Fprintf(mw, "foobar %d\n", i) - if i%13 == 0 { - if err := w.Flush(); err != nil { - return fmt.Errorf("error on flush: %w", err) - } - } - } - w.Close() - - zr := newReader(r) - data, err := ioutil.ReadAll(zr) - if err != nil { - return fmt.Errorf("unexpected error: %w, data=%q", err, data) - } - - wantData := wantW.Bytes() - if !bytes.Equal(data, wantData) { - return fmt.Errorf("unexpected data: %q. Expecting %q", data, wantData) - } - - return nil -} - -func testConcurrent(testFunc func() error, concurrency int) error { - ch := make(chan error, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - ch <- testFunc() - }() - } - for i := 0; i < concurrency; i++ { - select { - case err := <-ch: - if err != nil { - return fmt.Errorf("unexpected error on goroutine %d: %w", i, err) - } - case <-time.After(time.Second): - return fmt.Errorf("timeout on goroutine %d", i) - } - } - return nil -} diff --git a/lib/fasthttp/status_test.go b/lib/fasthttp/status_test.go deleted file mode 100644 index ff794a38e..000000000 --- a/lib/fasthttp/status_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package fasthttp - -import ( - "bytes" - "testing" -) - -func TestStatusLine(t *testing.T) { - t.Parallel() - - testStatusLine(t, -1, []byte("HTTP/1.1 -1 Unknown Status Code\r\n")) - testStatusLine(t, 99, []byte("HTTP/1.1 99 Unknown Status Code\r\n")) - testStatusLine(t, 200, []byte("HTTP/1.1 200 OK\r\n")) - testStatusLine(t, 512, []byte("HTTP/1.1 512 Unknown Status Code\r\n")) - testStatusLine(t, 512, []byte("HTTP/1.1 512 Unknown Status Code\r\n")) - testStatusLine(t, 520, []byte("HTTP/1.1 520 Unknown Status Code\r\n")) -} - -func testStatusLine(t *testing.T, statusCode int, expected []byte) { - line := formatStatusLine(nil, strHTTP11, statusCode, s2b(StatusMessage(statusCode))) - if !bytes.Equal(expected, line) { - t.Fatalf("unexpected status line %q. Expecting %q", string(line), string(expected)) - } -} diff --git a/lib/fasthttp/status_timing_test.go b/lib/fasthttp/status_timing_test.go deleted file mode 100644 index e35d8cef3..000000000 --- a/lib/fasthttp/status_timing_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package fasthttp - -import ( - "bytes" - "testing" -) - -func BenchmarkStatusLine99(b *testing.B) { - benchmarkStatusLine(b, 99, []byte("HTTP/1.1 99 Unknown Status Code\r\n")) -} - -func BenchmarkStatusLine200(b *testing.B) { - benchmarkStatusLine(b, 200, []byte("HTTP/1.1 200 OK\r\n")) -} - -func BenchmarkStatusLine512(b *testing.B) { - benchmarkStatusLine(b, 512, []byte("HTTP/1.1 512 Unknown Status Code\r\n")) -} - -func benchmarkStatusLine(b *testing.B, statusCode int, expected []byte) { - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - line := formatStatusLine(nil, strHTTP11, statusCode, s2b(StatusMessage(statusCode))) - if !bytes.Equal(expected, line) { - b.Fatalf("unexpected status line %q. Expecting %q", string(line), string(expected)) - } - } - }) -} diff --git a/lib/fasthttp/stream_test.go b/lib/fasthttp/stream_test.go deleted file mode 100644 index 2631e3d05..000000000 --- a/lib/fasthttp/stream_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package fasthttp - -import ( - "bufio" - "fmt" - "io" - "io/ioutil" - "testing" - "time" -) - -func TestNewStreamReader(t *testing.T) { - t.Parallel() - - ch := make(chan struct{}) - r := NewStreamReader(func(w *bufio.Writer) { - fmt.Fprintf(w, "Hello, world\n") - fmt.Fprintf(w, "Line #2\n") - close(ch) - }) - - data, err := ioutil.ReadAll(r) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - expectedData := "Hello, world\nLine #2\n" - if string(data) != expectedData { - t.Fatalf("unexpected data %q. Expecting %q", data, expectedData) - } - - if err = r.Close(); err != nil { - t.Fatalf("unexpected error") - } - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } -} - -func TestStreamReaderClose(t *testing.T) { - t.Parallel() - - firstLine := "the first line must pass" - ch := make(chan error, 1) - r := NewStreamReader(func(w *bufio.Writer) { - fmt.Fprintf(w, "%s", firstLine) - if err := w.Flush(); err != nil { - ch <- fmt.Errorf("unexpected error on first flush: %w", err) - return - } - - data := createFixedBody(4000) - for i := 0; i < 100; i++ { - w.Write(data) //nolint:errcheck - } - if err := w.Flush(); err == nil { - ch <- fmt.Errorf("expecting error on the second flush") - } - ch <- nil - }) - - buf := make([]byte, len(firstLine)) - n, err := io.ReadFull(r, buf) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if n != len(buf) { - t.Fatalf("unexpected number of bytes read: %d. Expecting %d", n, len(buf)) - } - if string(buf) != firstLine { - t.Fatalf("unexpected result: %q. Expecting %q", buf, firstLine) - } - - if err := r.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - select { - case err := <-ch: - if err != nil { - t.Fatalf("error returned from stream reader: %v", err) - } - case <-time.After(time.Second): - t.Fatalf("timeout when waiting for stream reader") - } - - // read trailing data - go func() { - if _, err := ioutil.ReadAll(r); err != nil { - ch <- fmt.Errorf("unexpected error when reading trailing data: %w", err) - return - } - ch <- nil - }() - - select { - case err := <-ch: - if err != nil { - t.Fatalf("error returned when reading tail data: %v", err) - } - case <-time.After(time.Second): - t.Fatalf("timeout when reading tail data") - } -} diff --git a/lib/fasthttp/stream_timing_test.go b/lib/fasthttp/stream_timing_test.go deleted file mode 100644 index b7fe03f9a..000000000 --- a/lib/fasthttp/stream_timing_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package fasthttp - -import ( - "bufio" - "io" - "testing" - "time" -) - -func BenchmarkStreamReader1(b *testing.B) { - benchmarkStreamReader(b, 1) -} - -func BenchmarkStreamReader10(b *testing.B) { - benchmarkStreamReader(b, 10) -} - -func BenchmarkStreamReader100(b *testing.B) { - benchmarkStreamReader(b, 100) -} - -func BenchmarkStreamReader1K(b *testing.B) { - benchmarkStreamReader(b, 1000) -} - -func BenchmarkStreamReader10K(b *testing.B) { - benchmarkStreamReader(b, 10000) -} - -func benchmarkStreamReader(b *testing.B, size int) { - src := createFixedBody(size) - b.SetBytes(int64(size)) - - b.RunParallel(func(pb *testing.PB) { - dst := make([]byte, size) - ch := make(chan error, 1) - sr := NewStreamReader(func(w *bufio.Writer) { - for pb.Next() { - if _, err := w.Write(src); err != nil { - ch <- err - return - } - if err := w.Flush(); err != nil { - ch <- err - return - } - } - ch <- nil - }) - for { - if _, err := sr.Read(dst); err != nil { - if err == io.EOF { - break - } - b.Fatalf("unexpected error when reading from stream reader: %v", err) - } - } - if err := sr.Close(); err != nil { - b.Fatalf("unexpected error when closing stream reader: %v", err) - } - select { - case err := <-ch: - if err != nil { - b.Fatalf("unexpected error from stream reader: %v", err) - } - case <-time.After(time.Second): - b.Fatalf("timeout") - } - }) -} diff --git a/lib/fasthttp/streaming_test.go b/lib/fasthttp/streaming_test.go deleted file mode 100644 index b451afb65..000000000 --- a/lib/fasthttp/streaming_test.go +++ /dev/null @@ -1,262 +0,0 @@ -package fasthttp - -import ( - "bufio" - "bytes" - "fmt" - "io/ioutil" - "os" - "sync" - "testing" - "time" - - "infini.sh/framework/lib/fasthttp/fasthttputil" -) - -func TestStreamingPipeline(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - reqS := `POST /one HTTP/1.1 -Host: example.com -Content-Length: 10 - -aaaaaaaaaa -POST /two HTTP/1.1 -Host: example.com -Content-Length: 10 - -aaaaaaaaaa` - - ln := fasthttputil.NewInmemoryListener() - - s := &Server{ - StreamRequestBody: true, - Handler: func(ctx *RequestCtx) { - body := "" - expected := "aaaaaaaaaa" - if string(ctx.Path()) == "/one" { - body = string(ctx.PostBody()) - } else { - all, err := ioutil.ReadAll(ctx.RequestBodyStream()) - if err != nil { - t.Error(err) - } - body = string(all) - } - if body != expected { - t.Errorf("expected %q got %q", expected, body) - } - }, - } - - ch := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(ch) - }() - - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if _, err = conn.Write([]byte(reqS)); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - var resp Response - br := bufio.NewReader(conn) - respCh := make(chan struct{}) - go func() { - if err := resp.Read(br); err != nil { - t.Errorf("error when reading response: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code %d. Expecting %d", resp.StatusCode(), StatusOK) - } - - if err := resp.Read(br); err != nil { - t.Errorf("error when reading response: %v", err) - } - if resp.StatusCode() != StatusOK { - t.Errorf("unexpected status code %d. Expecting %d", resp.StatusCode(), StatusOK) - } - close(respCh) - }() - - select { - case <-respCh: - case <-time.After(time.Second): - t.Fatal("timeout") - } - - if err := ln.Close(); err != nil { - t.Fatalf("error when closing listener: %v", err) - } - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout when waiting for the server to stop") - } -} - -func getChunkedTestEnv(t testing.TB) (*fasthttputil.InmemoryListener, []byte) { - body := createFixedBody(128 * 1024) - chunkedBody := createChunkedBody(body, nil, true) - - testHandler := func(ctx *RequestCtx) { - bodyBytes, err := ioutil.ReadAll(ctx.RequestBodyStream()) - if err != nil { - t.Logf("ioutil read returned err=%v", err) - t.Error("unexpected error while reading request body stream") - } - - if !bytes.Equal(body, bodyBytes) { - t.Errorf("unexpected request body, expected %q, got %q", body, bodyBytes) - } - } - s := &Server{ - Handler: testHandler, - StreamRequestBody: true, - MaxRequestBodySize: 1, // easier to test with small limit - } - - ln := fasthttputil.NewInmemoryListener() - - go func() { - err := s.Serve(ln) - if err != nil { - t.Errorf("could not serve listener: %v", err) - } - }() - - req := Request{} - req.SetHost("localhost") - req.Header.SetMethod("POST") - req.Header.Set("transfer-encoding", "chunked") - req.Header.SetContentLength(-1) - - formattedRequest := req.Header.Header() - formattedRequest = append(formattedRequest, chunkedBody...) - - return ln, formattedRequest -} - -func TestRequestStreamChunkedWithTrailer(t *testing.T) { - t.Parallel() - - body := createFixedBody(10) - expectedTrailer := map[string]string{ - "Foo": "footest", - "Bar": "bartest", - } - chunkedBody := createChunkedBody(body, expectedTrailer, true) - req := fmt.Sprintf(`POST / HTTP/1.1 -Host: example.com -Transfer-Encoding: chunked -Trailer: Foo, Bar - -%s -`, chunkedBody) - - ln := fasthttputil.NewInmemoryListener() - s := &Server{ - StreamRequestBody: true, - Handler: func(ctx *RequestCtx) { - all, err := ioutil.ReadAll(ctx.RequestBodyStream()) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if !bytes.Equal(all, body) { - t.Errorf("unexpected body %q. Expecting %q", all, body) - } - - for k, v := range expectedTrailer { - r := ctx.Request.Header.Peek(k) - if string(r) != v { - t.Errorf("unexpected trailer %q. Expecting %q. Got %q", k, v, r) - } - } - }, - } - - ch := make(chan struct{}) - go func() { - if err := s.Serve(ln); err != nil { - t.Errorf("unexpected error: %v", err) - } - close(ch) - }() - - conn, err := ln.Dial() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if _, err = conn.Write([]byte(req)); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if err := ln.Close(); err != nil { - t.Fatalf("error when closing listener: %v", err) - } - - select { - case <-ch: - case <-time.After(time.Second): - t.Fatal("timeout when waiting for the server to stop") - } -} - -func TestRequestStream(t *testing.T) { - t.Parallel() - - ln, formattedRequest := getChunkedTestEnv(t) - - c, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error while dialing: %v", err) - } - if _, err = c.Write(formattedRequest); err != nil { - t.Errorf("unexpected error while writing request: %v", err) - } - - br := bufio.NewReader(c) - var respH ResponseHeader - if err = respH.Read(br); err != nil { - t.Errorf("unexpected error: %v", err) - } -} - -func BenchmarkRequestStreamE2E(b *testing.B) { - ln, formattedRequest := getChunkedTestEnv(b) - - wg := &sync.WaitGroup{} - wg.Add(4) - for i := 0; i < 4; i++ { - go func(wg *sync.WaitGroup) { - for i := 0; i < b.N/4; i++ { - c, err := ln.Dial() - if err != nil { - b.Errorf("unexpected error while dialing: %v", err) - } - if _, err = c.Write(formattedRequest); err != nil { - b.Errorf("unexpected error while writing request: %v", err) - } - - br := bufio.NewReaderSize(c, 128) - var respH ResponseHeader - if err = respH.Read(br); err != nil { - b.Errorf("unexpected error: %v", err) - } - c.Close() - } - wg.Done() - }(wg) - } - - wg.Wait() -} diff --git a/lib/fasthttp/uri_test.go b/lib/fasthttp/uri_test.go deleted file mode 100644 index 650ee21ed..000000000 --- a/lib/fasthttp/uri_test.go +++ /dev/null @@ -1,490 +0,0 @@ -package fasthttp - -import ( - "bytes" - "fmt" - "os" - "reflect" - "runtime" - "testing" - "time" -) - -func TestURICopyToQueryArgs(t *testing.T) { - t.Parallel() - - var u URI - a := u.QueryArgs() - a.Set("foo", "bar") - - var u1 URI - u.CopyTo(&u1) - a1 := u1.QueryArgs() - - if string(a1.Peek("foo")) != "bar" { - t.Fatalf("unexpected query args value %q. Expecting %q", a1.Peek("foo"), "bar") - } -} - -func TestURIAcquireReleaseSequential(t *testing.T) { - t.Parallel() - - testURIAcquireRelease(t) -} - -func TestURIAcquireReleaseConcurrent(t *testing.T) { - t.Parallel() - - ch := make(chan struct{}, 10) - for i := 0; i < 10; i++ { - go func() { - testURIAcquireRelease(t) - ch <- struct{}{} - }() - } - - for i := 0; i < 10; i++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } -} - -func testURIAcquireRelease(t *testing.T) { - for i := 0; i < 10; i++ { - u := AcquireURI() - host := fmt.Sprintf("host.%d.com", i*23) - path := fmt.Sprintf("/foo/%d/bar", i*17) - queryArgs := "?foo=bar&baz=aass" - u.Parse([]byte(host), []byte(path+queryArgs)) //nolint:errcheck - if string(u.Host()) != host { - t.Fatalf("unexpected host %q. Expecting %q", u.Host(), host) - } - if string(u.Path()) != path { - t.Fatalf("unexpected path %q. Expecting %q", u.Path(), path) - } - ReleaseURI(u) - } -} - -func TestURILastPathSegment(t *testing.T) { - t.Parallel() - - testURILastPathSegment(t, "", "") - testURILastPathSegment(t, "/", "") - testURILastPathSegment(t, "/foo/bar/", "") - testURILastPathSegment(t, "/foobar.js", "foobar.js") - testURILastPathSegment(t, "/foo/bar/baz.html", "baz.html") -} - -func testURILastPathSegment(t *testing.T, path, expectedSegment string) { - var u URI - u.SetPath(path) - segment := u.LastPathSegment() - if string(segment) != expectedSegment { - t.Fatalf("unexpected last path segment for path %q: %q. Expecting %q", path, segment, expectedSegment) - } -} - -func TestURIPathEscape(t *testing.T) { - t.Parallel() - - testURIPathEscape(t, "/foo/bar", "/foo/bar") - testURIPathEscape(t, "/f_o-o=b:ar,b.c&q", "/f_o-o=b:ar,b.c&q") - testURIPathEscape(t, "/aa?bb.тест~qq", "/aa%3Fbb.%D1%82%D0%B5%D1%81%D1%82~qq") -} - -func testURIPathEscape(t *testing.T, path, expectedRequestURI string) { - var u URI - u.SetPath(path) - requestURI := u.RequestURI() - if string(requestURI) != expectedRequestURI { - t.Fatalf("unexpected requestURI %q. Expecting %q. path %q", requestURI, expectedRequestURI, path) - } -} - -func TestURIUpdate(t *testing.T) { - t.Parallel() - - // full uri - testURIUpdate(t, "http://example.net/dir/path1.html?param1=val1#fragment1", "https://example.com/dir/path2.html", "https://example.com/dir/path2.html") - - // empty uri - testURIUpdate(t, "http://example.com/dir/path1.html?param1=val1#fragment1", "", "http://example.com/dir/path1.html?param1=val1#fragment1") - - // request uri - testURIUpdate(t, "http://example.com/dir/path1.html?param1=val1#fragment1", "/dir/path2.html?param2=val2#fragment2", "http://example.com/dir/path2.html?param2=val2#fragment2") - - // schema - testURIUpdate(t, "http://example.com/dir/path1.html?param1=val1#fragment1", "https://example.com/dir/path1.html?param1=val1#fragment1", "https://example.com/dir/path1.html?param1=val1#fragment1") - - // relative uri - testURIUpdate(t, "http://example.com/baz/xxx.html?aaa=22#aaa", "bb.html?xx=12#pp", "http://example.com/baz/bb.html?xx=12#pp") - - testURIUpdate(t, "http://example.com/aaa.html?foo=bar", "?baz=434&aaa#xcv", "http://example.com/aaa.html?baz=434&aaa#xcv") - testURIUpdate(t, "http://example.com/baz", "~a/%20b=c,тест?йцу=ке", "http://example.com/~a/%20b=c,%D1%82%D0%B5%D1%81%D1%82?йцу=ке") - testURIUpdate(t, "http://example.com/baz", "/qwe#fragment", "http://example.com/qwe#fragment") - testURIUpdate(t, "http://example.com/baz/xxx", "aaa.html#bb?cc=dd&ee=dfd", "http://example.com/baz/aaa.html#bb?cc=dd&ee=dfd") - - if runtime.GOOS != "windows" { - testURIUpdate(t, "http://example.com/a/b/c/d", "../qwe/p?zx=34", "http://example.com/a/b/qwe/p?zx=34") - } - - // hash - testURIUpdate(t, "http://example.com/#fragment1", "#fragment2", "http://example.com/#fragment2") - - // uri without scheme - testURIUpdate(t, "https://example.net/dir/path1.html", "//example.com/dir/path2.html", "https://example.com/dir/path2.html") - testURIUpdate(t, "http://example.net/dir/path1.html", "//example.com/dir/path2.html", "http://example.com/dir/path2.html") - // host with port - testURIUpdate(t, "http://example.net/", "//example.com:8080/", "http://example.com:8080/") -} - -func testURIUpdate(t *testing.T, base, update, result string) { - var u URI - u.Parse(nil, []byte(base)) //nolint:errcheck - u.Update(update) - s := u.String() - if s != result { - t.Fatalf("unexpected result %q. Expecting %q. base=%q, update=%q", s, result, base, update) - } -} - -func TestURIPathNormalize(t *testing.T) { - if runtime.GOOS == "windows" { - t.SkipNow() - } - - t.Parallel() - - var u URI - - // double slash - testURIPathNormalize(t, &u, "/aa//bb", "/aa/bb") - - // triple slash - testURIPathNormalize(t, &u, "/x///y/", "/x/y/") - - // multi slashes - testURIPathNormalize(t, &u, "/abc//de///fg////", "/abc/de/fg/") - - // encoded slashes - testURIPathNormalize(t, &u, "/xxxx%2fyyy%2f%2F%2F", "/xxxx/yyy/") - - // dotdot - testURIPathNormalize(t, &u, "/aaa/..", "/") - - // dotdot with trailing slash - testURIPathNormalize(t, &u, "/xxx/yyy/../", "/xxx/") - - // multi dotdots - testURIPathNormalize(t, &u, "/aaa/bbb/ccc/../../ddd", "/aaa/ddd") - - // dotdots separated by other data - testURIPathNormalize(t, &u, "/a/b/../c/d/../e/..", "/a/c/") - - // too many dotdots - testURIPathNormalize(t, &u, "/aaa/../../../../xxx", "/xxx") - testURIPathNormalize(t, &u, "/../../../../../..", "/") - testURIPathNormalize(t, &u, "/../../../../../../", "/") - - // encoded dotdots - testURIPathNormalize(t, &u, "/aaa%2Fbbb%2F%2E.%2Fxxx", "/aaa/xxx") - - // double slash with dotdots - testURIPathNormalize(t, &u, "/aaa////..//b", "/b") - - // fake dotdot - testURIPathNormalize(t, &u, "/aaa/..bbb/ccc/..", "/aaa/..bbb/") - - // single dot - testURIPathNormalize(t, &u, "/a/./b/././c/./d.html", "/a/b/c/d.html") - testURIPathNormalize(t, &u, "./foo/", "/foo/") - testURIPathNormalize(t, &u, "./../.././../../aaa/bbb/../../../././../", "/") - testURIPathNormalize(t, &u, "./a/./.././../b/./foo.html", "/b/foo.html") -} - -func testURIPathNormalize(t *testing.T, u *URI, requestURI, expectedPath string) { - u.Parse(nil, []byte(requestURI)) //nolint:errcheck - if string(u.Path()) != expectedPath { - t.Fatalf("Unexpected path %q. Expected %q. requestURI=%q", u.Path(), expectedPath, requestURI) - } -} - -func TestURINoNormalization(t *testing.T) { - t.Parallel() - - var u URI - irregularPath := "/aaa%2Fbbb%2F%2E.%2Fxxx" - u.Parse(nil, []byte(irregularPath)) //nolint:errcheck - u.DisablePathNormalizing = true - if string(u.RequestURI()) != irregularPath { - t.Fatalf("Unexpected path %q. Expected %q.", u.Path(), irregularPath) - } -} - -func TestURICopyTo(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - t.Parallel() - - var u URI - var copyU URI - u.CopyTo(©U) - if !reflect.DeepEqual(u, copyU) { //nolint:govet - t.Fatalf("URICopyTo fail, u: \n%+v\ncopyu: \n%+v\n", u, copyU) //nolint:govet - } - - u.UpdateBytes([]byte("https://example.com/foo?bar=baz&baraz#qqqq")) - u.CopyTo(©U) - if !reflect.DeepEqual(u, copyU) { //nolint:govet - t.Fatalf("URICopyTo fail, u: \n%+v\ncopyu: \n%+v\n", u, copyU) //nolint:govet - } - -} - -func TestURIFullURI(t *testing.T) { - t.Parallel() - - var args Args - - // empty scheme, path and hash - testURIFullURI(t, "", "example.com", "", "", &args, "http://example.com/") - - // empty scheme and hash - testURIFullURI(t, "", "example.com", "/foo/bar", "", &args, "http://example.com/foo/bar") - - // empty hash - testURIFullURI(t, "fTP", "example.com", "/foo", "", &args, "ftp://example.com/foo") - - // empty args - testURIFullURI(t, "https", "example.com", "/", "aaa", &args, "https://example.com/#aaa") - - // non-empty args and non-ASCII path - args.Set("foo", "bar") - args.Set("xxx", "йух") - testURIFullURI(t, "", "example.com", "/тест123", "2er", &args, "http://example.com/%D1%82%D0%B5%D1%81%D1%82123?foo=bar&xxx=%D0%B9%D1%83%D1%85#2er") - - // test with empty args and non-empty query string - var u URI - u.Parse([]byte("example.com"), []byte("/foo?bar=baz&baraz#qqqq")) //nolint:errcheck - uri := u.FullURI() - expectedURI := "http://example.com/foo?bar=baz&baraz#qqqq" - if string(uri) != expectedURI { - t.Fatalf("Unexpected URI: %q. Expected %q", uri, expectedURI) - } -} - -func testURIFullURI(t *testing.T, scheme, host, path, hash string, args *Args, expectedURI string) { - var u URI - - u.SetScheme(scheme) - u.SetHost(host) - u.SetPath(path) - u.SetHash(hash) - args.CopyTo(u.QueryArgs()) - - uri := u.FullURI() - if string(uri) != expectedURI { - t.Fatalf("Unexpected URI: %q. Expected %q", uri, expectedURI) - } -} - -func TestURIParseNilHost(t *testing.T) { - t.Parallel() - - testURIParseScheme(t, "http://example.com/foo?bar#baz", "http", "example.com", "/foo?bar", "baz") - testURIParseScheme(t, "HTtP://example.com/", "http", "example.com", "/", "") - testURIParseScheme(t, "://example.com/xyz", "http", "example.com", "/xyz", "") - testURIParseScheme(t, "//example.com/foobar", "http", "example.com", "/foobar", "") - testURIParseScheme(t, "fTP://example.com", "ftp", "example.com", "/", "") - testURIParseScheme(t, "httPS://example.com", "https", "example.com", "/", "") - - // missing slash after hostname - testURIParseScheme(t, "http://example.com?baz=111", "http", "example.com", "/?baz=111", "") - - // slash in args - testURIParseScheme(t, "http://example.com?baz=111/222/xyz", "http", "example.com", "/?baz=111/222/xyz", "") - testURIParseScheme(t, "http://example.com?111/222/xyz", "http", "example.com", "/?111/222/xyz", "") -} - -func testURIParseScheme(t *testing.T, uri, expectedScheme, expectedHost, expectedRequestURI, expectedHash string) { - var u URI - u.Parse(nil, []byte(uri)) //nolint:errcheck - if string(u.Scheme()) != expectedScheme { - t.Fatalf("Unexpected scheme %q. Expecting %q for uri %q", u.Scheme(), expectedScheme, uri) - } - if string(u.Host()) != expectedHost { - t.Fatalf("Unexepcted host %q. Expecting %q for uri %q", u.Host(), expectedHost, uri) - } - if string(u.RequestURI()) != expectedRequestURI { - t.Fatalf("Unexepcted requestURI %q. Expecting %q for uri %q", u.RequestURI(), expectedRequestURI, uri) - } - if string(u.hash) != expectedHash { - t.Fatalf("Unexepcted hash %q. Expecting %q for uri %q", u.hash, expectedHash, uri) - } -} - -func TestIsHttp(t *testing.T) { - var u URI - if !u.isHttp() || u.isHttps() { - t.Fatalf("http scheme is assumed by default and not https") - } - u.SetSchemeBytes([]byte{}) - if !u.isHttp() || u.isHttps() { - t.Fatalf("empty scheme must be threaten as http and not https") - } - u.SetScheme("http") - if !u.isHttp() || u.isHttps() { - t.Fatalf("scheme must be threaten as http and not https") - } - u.SetScheme("https") - if !u.isHttps() || u.isHttp() { - t.Fatalf("scheme must be threaten as https and not http") - } - u.SetScheme("dav") - if u.isHttps() || u.isHttp() { - t.Fatalf("scheme must be threaten as not http and not https") - } -} - -func TestURIParse(t *testing.T) { - t.Parallel() - - var u URI - - // no args - testURIParse(t, &u, "example.com", "sdfdsf", - "http://example.com/sdfdsf", "example.com", "/sdfdsf", "sdfdsf", "", "") - - // args - testURIParse(t, &u, "example.com", "/aa?ss", - "http://example.com/aa?ss", "example.com", "/aa", "/aa", "ss", "") - - // args and hash - testURIParse(t, &u, "example.com", "/a.b.c?def=gkl#mnop", - "http://example.com/a.b.c?def=gkl#mnop", "example.com", "/a.b.c", "/a.b.c", "def=gkl", "mnop") - - // '?' and '#' in hash - testURIParse(t, &u, "example.com", "/foo#bar?baz=aaa#bbb", - "http://example.com/foo#bar?baz=aaa#bbb", "example.com", "/foo", "/foo", "", "bar?baz=aaa#bbb") - - // encoded path - testURIParse(t, &u, "example.com", "/Test%20+%20%D0%BF%D1%80%D0%B8?asdf=%20%20&s=12#sdf", - "http://example.com/Test%20+%20%D0%BF%D1%80%D0%B8?asdf=%20%20&s=12#sdf", "example.com", "/Test + при", "/Test%20+%20%D0%BF%D1%80%D0%B8", "asdf=%20%20&s=12", "sdf") - - // host in uppercase - testURIParse(t, &u, "example.com", "/bC?De=F#Gh", - "http://example.com/bC?De=F#Gh", "example.com", "/bC", "/bC", "De=F", "Gh") - - // uri with hostname - testURIParse(t, &u, "example.com", "http://example.com/foo/bar?baz=aaa#ddd", - "http://example.com/foo/bar?baz=aaa#ddd", "example.com", "/foo/bar", "/foo/bar", "baz=aaa", "ddd") - testURIParse(t, &u, "example.net", "https://example.com/f/b%20r?baz=aaa#ddd", - "https://example.com/f/b%20r?baz=aaa#ddd", "example.com", "/f/b r", "/f/b%20r", "baz=aaa", "ddd") - - // no slash after hostname in uri - testURIParse(t, &u, "example.com", "http://example.com", - "http://example.com/", "example.com", "/", "/", "", "") - - // uppercase hostname in uri - testURIParse(t, &u, "example.net", "http://EXAMPLE.COM/aaa", - "http://example.com/aaa", "example.com", "/aaa", "/aaa", "", "") - - // http:// in query params - testURIParse(t, &u, "example.com", "/foo?bar=http://example.org", - "http://example.com/foo?bar=http://example.org", "example.com", "/foo", "/foo", "bar=http://example.org", "") - - testURIParse(t, &u, "example.com", "//relative", - "http://example.com/relative", "example.com", "/relative", "//relative", "", "") - - testURIParse(t, &u, "", "//example.com//absolute", - "http://example.com/absolute", "example.com", "/absolute", "//absolute", "", "") - - testURIParse(t, &u, "", "//example.com\r\n\r\nGET x", - "http:///", "", "/", "", "", "") - - testURIParse(t, &u, "", "http://[fe80::1%25en0]/", - "http://[fe80::1%en0]/", "[fe80::1%en0]", "/", "/", "", "") - - testURIParse(t, &u, "", "http://[fe80::1%25en0]:8080/", - "http://[fe80::1%en0]:8080/", "[fe80::1%en0]:8080", "/", "/", "", "") - - testURIParse(t, &u, "", "http://hello.世界.com/foo", - "http://hello.世界.com/foo", "hello.世界.com", "/foo", "/foo", "", "") - - testURIParse(t, &u, "", "http://hello.%e4%b8%96%e7%95%8c.com/foo", - "http://hello.世界.com/foo", "hello.世界.com", "/foo", "/foo", "", "") -} - -func testURIParse(t *testing.T, u *URI, host, uri, - expectedURI, expectedHost, expectedPath, expectedPathOriginal, expectedArgs, expectedHash string) { - u.Parse([]byte(host), []byte(uri)) //nolint:errcheck - - if !bytes.Equal(u.FullURI(), []byte(expectedURI)) { - t.Fatalf("Unexpected uri %q. Expected %q. host=%q, uri=%q", u.FullURI(), expectedURI, host, uri) - } - if !bytes.Equal(u.Host(), []byte(expectedHost)) { - t.Fatalf("Unexpected host %q. Expected %q. host=%q, uri=%q", u.Host(), expectedHost, host, uri) - } - if !bytes.Equal(u.PathOriginal(), []byte(expectedPathOriginal)) { - t.Fatalf("Unexpected original path %q. Expected %q. host=%q, uri=%q", u.PathOriginal(), expectedPathOriginal, host, uri) - } - if !bytes.Equal(u.Path(), []byte(expectedPath)) { - t.Fatalf("Unexpected path %q. Expected %q. host=%q, uri=%q", u.Path(), expectedPath, host, uri) - } - if !bytes.Equal(u.QueryString(), []byte(expectedArgs)) { - t.Fatalf("Unexpected args %q. Expected %q. host=%q, uri=%q", u.QueryString(), expectedArgs, host, uri) - } - if !bytes.Equal(u.Hash(), []byte(expectedHash)) { - t.Fatalf("Unexpected hash %q. Expected %q. host=%q, uri=%q", u.Hash(), expectedHash, host, uri) - } -} - -func TestURIWithQuerystringOverride(t *testing.T) { - t.Parallel() - - var u URI - u.SetQueryString("q1=foo&q2=bar") - u.QueryArgs().Add("q3", "baz") - u.SetQueryString("q1=foo&q2=bar&q4=quux") - uriString := string(u.RequestURI()) - - if uriString != "/?q1=foo&q2=bar&q4=quux" { - t.Fatalf("Expected Querystring to be overridden but was %q ", uriString) - } -} - -func TestInvalidUrl(t *testing.T) { - url := `https://.çèéà@&~!&:=\\/\"'~<>|+-*()[]{}%$;,¥&&$22|||<>< 4ly8lzjmoNx233AXELDtyaFQiiUH-fd8c-CnXUJVYnGIs4Uwr-bptom5GCnWtsGMQxeM2ZhoKE973eKgs2Sjh6RePnyaLpCi6SiNSLevcMoraARrp88L-SgtKqd-XHAtSI8hiPRiXPQmDIA4BGhSgoc0nfn1PoYuGKKmDcZ04tANRc3iz4aF4-A1UrO8bLHTH7MEJvzx.someqa.fr/A/?&QS_BEGIN<&8{b'Ob=p*f> QS_END` - - u := AcquireURI() - defer ReleaseURI(u) - - if err := u.Parse(nil, []byte(url)); err == nil { - t.Fail() - } -} - -func TestNoOverwriteInput(t *testing.T) { - str := `//%AA` - url := []byte(str) - - u := AcquireURI() - defer ReleaseURI(u) - - if err := u.Parse(nil, url); err != nil { - t.Error(err) - } - - if string(url) != str { - t.Error() - } - - if u.String() != "http://\xaa/" { - t.Errorf("%q", u.String()) - } -} diff --git a/lib/fasthttp/uri_timing_test.go b/lib/fasthttp/uri_timing_test.go deleted file mode 100644 index 1d2cdd46c..000000000 --- a/lib/fasthttp/uri_timing_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package fasthttp - -import ( - "testing" -) - -func BenchmarkURIParsePath(b *testing.B) { - benchmarkURIParse(b, "google.com", "/foo/bar") -} - -func BenchmarkURIParsePathQueryString(b *testing.B) { - benchmarkURIParse(b, "google.com", "/foo/bar?query=string&other=value") -} - -func BenchmarkURIParsePathQueryStringHash(b *testing.B) { - benchmarkURIParse(b, "google.com", "/foo/bar?query=string&other=value#hashstring") -} - -func BenchmarkURIParseHostname(b *testing.B) { - benchmarkURIParse(b, "google.com", "http://foobar.com/foo/bar?query=string&other=value#hashstring") -} - -func BenchmarkURIFullURI(b *testing.B) { - host := []byte("foobar.com") - requestURI := []byte("/foobar/baz?aaa=bbb&ccc=ddd") - uriLen := len(host) + len(requestURI) + 7 - - b.RunParallel(func(pb *testing.PB) { - var u URI - u.Parse(host, requestURI) //nolint:errcheck - for pb.Next() { - uri := u.FullURI() - if len(uri) != uriLen { - b.Fatalf("unexpected uri len %d. Expecting %d", len(uri), uriLen) - } - } - }) -} - -func benchmarkURIParse(b *testing.B, host, uri string) { - strHost, strURI := []byte(host), []byte(uri) - - b.RunParallel(func(pb *testing.PB) { - var u URI - for pb.Next() { - u.Parse(strHost, strURI) //nolint:errcheck - } - }) -} diff --git a/lib/fasthttp/uri_windows_test.go b/lib/fasthttp/uri_windows_test.go deleted file mode 100644 index d26dfdfc1..000000000 --- a/lib/fasthttp/uri_windows_test.go +++ /dev/null @@ -1,15 +0,0 @@ -//go:build windows -// +build windows - -package fasthttp - -import "testing" - -func TestURIPathNormalizeIssue86(t *testing.T) { - t.Parallel() - - // see https://infini.sh/framework/lib/fasthttp/issues/86 - var u URI - - testURIPathNormalize(t, &u, `C:\a\b\c\fs.go`, `C:\a\b\c\fs.go`) -} diff --git a/lib/fasthttp/userdata_test.go b/lib/fasthttp/userdata_test.go deleted file mode 100644 index 3a081aeea..000000000 --- a/lib/fasthttp/userdata_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package fasthttp - -import ( - "fmt" - "reflect" - "testing" -) - -func TestUserData(t *testing.T) { - t.Parallel() - - var u userData - - for i := 0; i < 10; i++ { - key := []byte(fmt.Sprintf("key_%d", i)) - u.SetBytes(key, i+5) - testUserDataGet(t, &u, key, i+5) - u.SetBytes(key, i) - testUserDataGet(t, &u, key, i) - } - - for i := 0; i < 10; i++ { - key := []byte(fmt.Sprintf("key_%d", i)) - testUserDataGet(t, &u, key, i) - } - - u.Reset() - - for i := 0; i < 10; i++ { - key := []byte(fmt.Sprintf("key_%d", i)) - testUserDataGet(t, &u, key, nil) - } -} - -func testUserDataGet(t *testing.T, u *userData, key []byte, value interface{}) { - v := u.GetBytes(key) - if v == nil && value != nil { - t.Fatalf("cannot obtain value for key=%q", key) - } - if !reflect.DeepEqual(v, value) { - t.Fatalf("unexpected value for key=%q: %d. Expecting %d", key, v, value) - } -} - -func TestUserDataValueClose(t *testing.T) { - t.Parallel() - - var u userData - - closeCalls := 0 - - // store values implementing io.Closer - for i := 0; i < 5; i++ { - key := fmt.Sprintf("key_%d", i) - u.Set(key, &closerValue{&closeCalls}) - } - - // store values without io.Closer - for i := 0; i < 10; i++ { - key := fmt.Sprintf("key_noclose_%d", i) - u.Set(key, i) - } - - u.Reset() - - if closeCalls != 5 { - t.Fatalf("unexpected number of Close calls: %d. Expecting 10", closeCalls) - } -} - -type closerValue struct { - closeCalls *int -} - -func (cv *closerValue) Close() error { - (*cv.closeCalls)++ - return nil -} - -func TestUserDataDelete(t *testing.T) { - t.Parallel() - - var u userData - - for i := 0; i < 10; i++ { - key := fmt.Sprintf("key_%d", i) - u.Set(key, i) - testUserDataGet(t, &u, []byte(key), i) - } - - for i := 0; i < 10; i += 2 { - k := fmt.Sprintf("key_%d", i) - u.Remove(k) - if val := u.Get(k); val != nil { - t.Fatalf("unexpected key= %q, value =%v ,Expecting key= %q, value = nil", k, val, k) - } - kk := fmt.Sprintf("key_%d", i+1) - testUserDataGet(t, &u, []byte(kk), i+1) - } - for i := 0; i < 10; i++ { - key := fmt.Sprintf("key_new_%d", i) - u.Set(key, i) - testUserDataGet(t, &u, []byte(key), i) - } - -} - -func TestUserDataSetAndRemove(t *testing.T) { - var ( - u userData - shortKey = "[]" - longKey = "[ ]" - ) - - u.Set(shortKey, "") - u.Set(longKey, "") - u.Remove(shortKey) - u.Set(shortKey, "") - testUserDataGet(t, &u, []byte(shortKey), "") - testUserDataGet(t, &u, []byte(longKey), "") -} diff --git a/lib/fasthttp/userdata_timing_test.go b/lib/fasthttp/userdata_timing_test.go deleted file mode 100644 index 3822de3fd..000000000 --- a/lib/fasthttp/userdata_timing_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package fasthttp - -import ( - "testing" -) - -func BenchmarkUserDataCustom(b *testing.B) { - keys := []string{"foobar", "baz", "aaa", "bsdfs"} - b.RunParallel(func(pb *testing.PB) { - var u userData - var v interface{} = u - for pb.Next() { - for _, key := range keys { - u.Set(key, v) - } - for _, key := range keys { - vv := u.Get(key) - if _, ok := vv.(userData); !ok { - b.Fatalf("unexpected value %v for key %q", vv, key) - } - } - u.Reset() - } - }) -} - -func BenchmarkUserDataStdMap(b *testing.B) { - keys := []string{"foobar", "baz", "aaa", "bsdfs"} - b.RunParallel(func(pb *testing.PB) { - u := make(map[string]interface{}) - var v interface{} = u - for pb.Next() { - for _, key := range keys { - u[key] = v - } - for _, key := range keys { - vv := u[key] - if _, ok := vv.(map[string]interface{}); !ok { - b.Fatalf("unexpected value %v for key %q", vv, key) - } - } - - for k := range u { - delete(u, k) - } - } - }) -} diff --git a/lib/fasthttp/workerpool_test.go b/lib/fasthttp/workerpool_test.go deleted file mode 100644 index 4e4617d96..000000000 --- a/lib/fasthttp/workerpool_test.go +++ /dev/null @@ -1,177 +0,0 @@ -package fasthttp - -import ( - "io/ioutil" - "net" - "testing" - "time" - - "infini.sh/framework/lib/fasthttp/fasthttputil" -) - -func TestWorkerPoolStartStopSerial(t *testing.T) { - t.Parallel() - - testWorkerPoolStartStop(t) -} - -func TestWorkerPoolStartStopConcurrent(t *testing.T) { - t.Parallel() - - concurrency := 10 - ch := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - testWorkerPoolStartStop(t) - ch <- struct{}{} - }() - } - for i := 0; i < concurrency; i++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } -} - -func testWorkerPoolStartStop(t *testing.T) { - wp := &workerPool{ - WorkerFunc: func(conn net.Conn) error { return nil }, - MaxWorkersCount: 10, - //Logger: defaultLogger, - } - for i := 0; i < 10; i++ { - wp.Start() - wp.Stop() - } -} - -func TestWorkerPoolMaxWorkersCountSerial(t *testing.T) { - t.Parallel() - - testWorkerPoolMaxWorkersCountMulti(t) -} - -func TestWorkerPoolMaxWorkersCountConcurrent(t *testing.T) { - t.Parallel() - - concurrency := 4 - ch := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - testWorkerPoolMaxWorkersCountMulti(t) - ch <- struct{}{} - }() - } - for i := 0; i < concurrency; i++ { - select { - case <-ch: - case <-time.After(time.Second * 2): - t.Fatalf("timeout") - } - } -} - -func testWorkerPoolMaxWorkersCountMulti(t *testing.T) { - for i := 0; i < 5; i++ { - testWorkerPoolMaxWorkersCount(t) - } -} - -func testWorkerPoolMaxWorkersCount(t *testing.T) { - ready := make(chan struct{}) - wp := &workerPool{ - WorkerFunc: func(conn net.Conn) error { - buf := make([]byte, 100) - n, err := conn.Read(buf) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - buf = buf[:n] - if string(buf) != "foobar" { - t.Errorf("unexpected data read: %q. Expecting %q", buf, "foobar") - } - if _, err = conn.Write([]byte("baz")); err != nil { - t.Errorf("unexpected error: %v", err) - } - - <-ready - - return nil - }, - MaxWorkersCount: 10, - //Logger: defaultLogger, - connState: func(net.Conn, ConnState) {}, - } - wp.Start() - - ln := fasthttputil.NewInmemoryListener() - - clientCh := make(chan struct{}, wp.MaxWorkersCount) - for i := 0; i < wp.MaxWorkersCount; i++ { - go func() { - conn, err := ln.Dial() - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if _, err = conn.Write([]byte("foobar")); err != nil { - t.Errorf("unexpected error: %v", err) - } - data, err := ioutil.ReadAll(conn) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - if string(data) != "baz" { - t.Errorf("unexpected value read: %q. Expecting %q", data, "baz") - } - if err = conn.Close(); err != nil { - t.Errorf("unexpected error: %v", err) - } - clientCh <- struct{}{} - }() - } - - for i := 0; i < wp.MaxWorkersCount; i++ { - conn, err := ln.Accept() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if !wp.Serve(conn) { - t.Fatalf("worker pool must have enough workers to serve the conn") - } - } - - go func() { - if _, err := ln.Dial(); err != nil { - t.Errorf("unexpected error: %v", err) - } - }() - conn, err := ln.Accept() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - for i := 0; i < 5; i++ { - if wp.Serve(conn) { - t.Fatalf("worker pool must be full") - } - } - if err = conn.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - - close(ready) - - for i := 0; i < wp.MaxWorkersCount; i++ { - select { - case <-clientCh: - case <-time.After(time.Second): - t.Fatalf("timeout") - } - } - - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %v", err) - } - wp.Stop() -} diff --git a/lib/fastjson_marshal/marshaler_test.go b/lib/fastjson_marshal/marshaler_test.go index 27382534c..d533ad336 100644 --- a/lib/fastjson_marshal/marshaler_test.go +++ b/lib/fastjson_marshal/marshaler_test.go @@ -3,7 +3,6 @@ package fastjson_marshal import ( "encoding/json" "errors" - "os" "reflect" "testing" ) @@ -65,61 +64,6 @@ func TestMarshalMarshaler(t *testing.T) { assertEncoded(t, &w, `"custom_logic"`) } -func TestMarshalAppender(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - var w Writer - err := Marshal(&w, appenderFunc(func(in []byte) []byte { - return append(in, `"appended"`...) - })) - if err != nil { - t.Fatal(err) - } - assertEncoded(t, &w, `"appended"`) -} - -func TestMarshalStdlibMarshaler(t *testing.T) { - var w Writer - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - Marshal(&w, stdlibMarshalerFunc(func() ([]byte, error) { - return []byte(`"json.Marshaled"`), nil - })) - assertEncoded(t, &w, `"json.Marshaled"`) -} - -func TestMarshalStdlibMarshalerPanic(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - var w Writer - err := Marshal(&w, stdlibMarshalerFunc(func() ([]byte, error) { - panic("boom") - })) - assertEncoded(t, &w, `{"__PANIC__":"panic calling MarshalJSON for type fastjson.stdlibMarshalerFunc: boom"}`) - expectedErr := `panic calling MarshalJSON for type fastjson.stdlibMarshalerFunc: boom` - if err == nil || err.Error() != expectedErr { - t.Fatalf("expected %q, got %q", expectedErr, err) - } -} - -func TestMarshalStdlibMarshalerError(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - var w Writer - err := Marshal(&w, stdlibMarshalerFunc(func() ([]byte, error) { - return nil, errors.New("boom") - })) - assertEncoded(t, &w, `{"__ERROR__":"json: error calling MarshalJSON for type fastjson.stdlibMarshalerFunc: boom"}`) - expectedErr := `json: error calling MarshalJSON for type fastjson.stdlibMarshalerFunc: boom` - if err == nil || err.Error() != expectedErr { - t.Fatalf("expected %q, got %q", expectedErr, err) - } -} - func TestMarshalMapValueError(t *testing.T) { var w Writer expectedErr := errors.New("nope") @@ -148,18 +92,6 @@ func (f marshalerFunc) MarshalFastJSON(w *Writer) error { return f(w) } -type appenderFunc func([]byte) []byte - -func (f appenderFunc) AppendJSON(in []byte) []byte { - return f(in) -} - -type stdlibMarshalerFunc func() ([]byte, error) - -func (f stdlibMarshalerFunc) MarshalJSON() ([]byte, error) { - return f() -} - func assertEncoded(t *testing.T, w *Writer, expected string) { actual := string(w.Bytes()) if actual != expected { diff --git a/lib/gomail/LICENSE b/lib/gomail/LICENSE new file mode 100644 index 000000000..5f5c12af7 --- /dev/null +++ b/lib/gomail/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Alexandre Cesaro + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/lib/gomail/README.md b/lib/gomail/README.md new file mode 100644 index 000000000..b3be9e146 --- /dev/null +++ b/lib/gomail/README.md @@ -0,0 +1,92 @@ +# Gomail +[![Build Status](https://travis-ci.org/go-gomail/gomail.svg?branch=v2)](https://travis-ci.org/go-gomail/gomail) [![Code Coverage](http://gocover.io/_badge/gopkg.in/gomail.v2)](http://gocover.io/gopkg.in/gomail.v2) [![Documentation](https://godoc.org/gopkg.in/gomail.v2?status.svg)](https://godoc.org/gopkg.in/gomail.v2) + +## Introduction + +Gomail is a simple and efficient package to send emails. It is well tested and +documented. + +Gomail can only send emails using an SMTP server. But the API is flexible and it +is easy to implement other methods for sending emails using a local Postfix, an +API, etc. + +It is versioned using [gopkg.in](https://gopkg.in) so I promise +there will never be backward incompatible changes within each version. + +It requires Go 1.2 or newer. With Go 1.5, no external dependencies are used. + + +## Features + +Gomail supports: +- Attachments +- Embedded images +- HTML and text templates +- Automatic encoding of special characters +- SSL and TLS +- Sending multiple emails with the same SMTP connection + + +## Documentation + +https://godoc.org/gopkg.in/gomail.v2 + + +## Download + + go get gopkg.in/gomail.v2 + + +## Examples + +See the [examples in the documentation](https://godoc.org/gopkg.in/gomail.v2#example-package). + + +## FAQ + +### x509: certificate signed by unknown authority + +If you get this error it means the certificate used by the SMTP server is not +considered valid by the client running Gomail. As a quick workaround you can +bypass the verification of the server's certificate chain and host name by using +`SetTLSConfig`: + + package main + + import ( + "crypto/tls" + + "gopkg.in/gomail.v2" + ) + + func main() { + d := gomail.NewDialer("smtp.example.com", 587, "user", "123456") + d.TLSConfig = &tls.Config{InsecureSkipVerify: true} + + // Send emails using d. + } + +Note, however, that this is insecure and should not be used in production. + + +## Contribute + +Contributions are more than welcome! See [CONTRIBUTING.md](CONTRIBUTING.md) for +more info. + + +## Change log + +See [CHANGELOG.md](CHANGELOG.md). + + +## License + +[MIT](LICENSE) + + +## Contact + +You can ask questions on the [Gomail +thread](https://groups.google.com/d/topic/golang-nuts/jMxZHzvvEVg/discussion) +in the Go mailing-list. diff --git a/lib/gomail/auth.go b/lib/gomail/auth.go new file mode 100644 index 000000000..d28b83ab7 --- /dev/null +++ b/lib/gomail/auth.go @@ -0,0 +1,49 @@ +package gomail + +import ( + "bytes" + "errors" + "fmt" + "net/smtp" +) + +// loginAuth is an smtp.Auth that implements the LOGIN authentication mechanism. +type loginAuth struct { + username string + password string + host string +} + +func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) { + if !server.TLS { + advertised := false + for _, mechanism := range server.Auth { + if mechanism == "LOGIN" { + advertised = true + break + } + } + if !advertised { + return "", nil, errors.New("gomail: unencrypted connection") + } + } + if server.Name != a.host { + return "", nil, errors.New("gomail: wrong host name") + } + return "LOGIN", nil, nil +} + +func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) { + if !more { + return nil, nil + } + + switch { + case bytes.Equal(fromServer, []byte("Username:")): + return []byte(a.username), nil + case bytes.Equal(fromServer, []byte("Password:")): + return []byte(a.password), nil + default: + return nil, fmt.Errorf("gomail: unexpected server challenge: %s", fromServer) + } +} diff --git a/lib/gomail/auth_test.go b/lib/gomail/auth_test.go new file mode 100644 index 000000000..428ef3467 --- /dev/null +++ b/lib/gomail/auth_test.go @@ -0,0 +1,100 @@ +package gomail + +import ( + "net/smtp" + "testing" +) + +const ( + testUser = "user" + testPwd = "pwd" + testHost = "smtp.example.com" +) + +type authTest struct { + auths []string + challenges []string + tls bool + wantData []string + wantError bool +} + +func TestNoAdvertisement(t *testing.T) { + testLoginAuth(t, &authTest{ + auths: []string{}, + tls: false, + wantError: true, + }) +} + +func TestNoAdvertisementTLS(t *testing.T) { + testLoginAuth(t, &authTest{ + auths: []string{}, + challenges: []string{"Username:", "Password:"}, + tls: true, + wantData: []string{"", testUser, testPwd}, + }) +} + +func TestLogin(t *testing.T) { + testLoginAuth(t, &authTest{ + auths: []string{"PLAIN", "LOGIN"}, + challenges: []string{"Username:", "Password:"}, + tls: false, + wantData: []string{"", testUser, testPwd}, + }) +} + +func TestLoginTLS(t *testing.T) { + testLoginAuth(t, &authTest{ + auths: []string{"LOGIN"}, + challenges: []string{"Username:", "Password:"}, + tls: true, + wantData: []string{"", testUser, testPwd}, + }) +} + +func testLoginAuth(t *testing.T, test *authTest) { + auth := &loginAuth{ + username: testUser, + password: testPwd, + host: testHost, + } + server := &smtp.ServerInfo{ + Name: testHost, + TLS: test.tls, + Auth: test.auths, + } + proto, toServer, err := auth.Start(server) + if err != nil && !test.wantError { + t.Fatalf("loginAuth.Start(): %v", err) + } + if err != nil && test.wantError { + return + } + if proto != "LOGIN" { + t.Errorf("invalid protocol, got %q, want LOGIN", proto) + } + + i := 0 + got := string(toServer) + if got != test.wantData[i] { + t.Errorf("Invalid response, got %q, want %q", got, test.wantData[i]) + } + + for _, challenge := range test.challenges { + i++ + if i >= len(test.wantData) { + t.Fatalf("unexpected challenge: %q", challenge) + } + + toServer, err = auth.Next([]byte(challenge), true) + if err != nil { + t.Fatalf("loginAuth.Auth(): %v", err) + } + got = string(toServer) + if got != test.wantData[i] { + t.Errorf("Invalid response, got %q, want %q", got, test.wantData[i]) + } + } +} diff --git a/lib/gomail/doc.go b/lib/gomail/doc.go new file mode 100644 index 000000000..a8f5091f5 --- /dev/null +++ b/lib/gomail/doc.go @@ -0,0 +1,5 @@ +// Package gomail provides a simple interface to compose emails and to mail them +// efficiently. +// +// More info on Github: https://github.com/go-gomail/gomail +package gomail diff --git a/lib/gomail/example_test.go b/lib/gomail/example_test.go new file mode 100644 index 000000000..a1e1790b2 --- /dev/null +++ b/lib/gomail/example_test.go @@ -0,0 +1,223 @@ +package gomail_test + +import ( + "fmt" + "html/template" + "io" + "log" + "time" + + "infini.sh/framework/lib/gomail" +) + +func Example() { + m := gomail.NewMessage() + m.SetHeader("From", "alex@example.com") + m.SetHeader("To", "bob@example.com", "cora@example.com") + m.SetAddressHeader("Cc", "dan@example.com", "Dan") + m.SetHeader("Subject", "Hello!") + m.SetBody("text/html", "Hello Bob and Cora!") + m.Attach("/home/Alex/lolcat.jpg") + + d := gomail.NewDialer("smtp.example.com", 587, "user", "123456") + + // Send the email to Bob, Cora and Dan. + if err := d.DialAndSend(m); err != nil { + panic(err) + } +} + +// A daemon that listens to a channel and sends all incoming messages. +func Example_daemon() { + ch := make(chan *gomail.Message) + + go func() { + d := gomail.NewDialer("smtp.example.com", 587, "user", "123456") + + var s gomail.SendCloser + var err error + open := false + for { + select { + case m, ok := <-ch: + if !ok { + return + } + if !open { + if s, err = d.Dial(); err != nil { + panic(err) + } + open = true + } + if err := gomail.Send(s, m); err != nil { + log.Print(err) + } + // Close the connection to the SMTP server if no email was sent in + // the last 30 seconds. + case <-time.After(30 * time.Second): + if open { + if err := s.Close(); err != nil { + panic(err) + } + open = false + } + } + } + }() + + // Use the channel in your program to send emails. + + // Close the channel to stop the mail daemon. + close(ch) +} + +// Efficiently send a customized newsletter to a list of recipients. +func Example_newsletter() { + // The list of recipients. + var list []struct { + Name string + Address string + } + + d := gomail.NewDialer("smtp.example.com", 587, "user", "123456") + s, err := d.Dial() + if err != nil { + panic(err) + } + + m := gomail.NewMessage() + for _, r := range list { + m.SetHeader("From", "no-reply@example.com") + m.SetAddressHeader("To", r.Address, r.Name) + m.SetHeader("Subject", "Newsletter #1") + m.SetBody("text/html", fmt.Sprintf("Hello %s!", r.Name)) + + if err := gomail.Send(s, m); err != nil { + log.Printf("Could not send email to %q: %v", r.Address, err) + } + m.Reset() + } +} + +// Send an email using a local SMTP server. +func Example_noAuth() { + m := gomail.NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetHeader("Subject", "Hello!") + m.SetBody("text/plain", "Hello!") + + d := gomail.Dialer{Host: "localhost", Port: 587} + if err := d.DialAndSend(m); err != nil { + panic(err) + } +} + +// Send an email using an API or postfix. +func Example_noSMTP() { + m := gomail.NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetHeader("Subject", "Hello!") + m.SetBody("text/plain", "Hello!") + + s := gomail.SendFunc(func(from string, to []string, msg io.WriterTo) error { + // Implements you email-sending function, for example by calling + // an API, or running postfix, etc. + fmt.Println("From:", from) + fmt.Println("To:", to) + return nil + }) + + if err := gomail.Send(s, m); err != nil { + panic(err) + } + // Output: + // From: from@example.com + // To: [to@example.com] +} + +var m *gomail.Message + +func ExampleSetCopyFunc() { + m.Attach("foo.txt", gomail.SetCopyFunc(func(w io.Writer) error { + _, err := w.Write([]byte("Content of foo.txt")) + return err + })) +} + +func ExampleSetHeader() { + h := map[string][]string{"Content-ID": {""}} + m.Attach("foo.jpg", gomail.SetHeader(h)) +} + +func ExampleRename() { + m.Attach("/tmp/0000146.jpg", gomail.Rename("picture.jpg")) +} + +func ExampleMessage_AddAlternative() { + m.SetBody("text/plain", "Hello!") + m.AddAlternative("text/html", "

Hello!

") +} + +func ExampleMessage_AddAlternativeWriter() { + t := template.Must(template.New("example").Parse("Hello {{.}}!")) + m.AddAlternativeWriter("text/plain", func(w io.Writer) error { + return t.Execute(w, "Bob") + }) +} + +func ExampleMessage_Attach() { + m.Attach("/tmp/image.jpg") +} + +func ExampleMessage_Embed() { + m.Embed("/tmp/image.jpg") + m.SetBody("text/html", `My image`) +} + +func ExampleMessage_FormatAddress() { + m.SetHeader("To", m.FormatAddress("bob@example.com", "Bob"), m.FormatAddress("cora@example.com", "Cora")) +} + +func ExampleMessage_FormatDate() { + m.SetHeaders(map[string][]string{ + "X-Date": {m.FormatDate(time.Now())}, + }) +} + +func ExampleMessage_SetAddressHeader() { + m.SetAddressHeader("To", "bob@example.com", "Bob") +} + +func ExampleMessage_SetBody() { + m.SetBody("text/plain", "Hello!") +} + +func ExampleMessage_SetDateHeader() { + m.SetDateHeader("X-Date", time.Now()) +} + +func ExampleMessage_SetHeader() { + m.SetHeader("Subject", "Hello!") +} + +func ExampleMessage_SetHeaders() { + m.SetHeaders(map[string][]string{ + "From": {m.FormatAddress("alex@example.com", "Alex")}, + "To": {"bob@example.com", "cora@example.com"}, + "Subject": {"Hello"}, + }) +} + +func ExampleSetCharset() { + m = gomail.NewMessage(gomail.SetCharset("ISO-8859-1")) +} + +func ExampleSetEncoding() { + m = gomail.NewMessage(gomail.SetEncoding(gomail.Base64)) +} + +func ExampleSetPartEncoding() { + m.SetBody("text/plain", "Hello!", gomail.SetPartEncoding(gomail.Unencoded)) +} diff --git a/lib/gomail/message.go b/lib/gomail/message.go new file mode 100644 index 000000000..4bffb1e7f --- /dev/null +++ b/lib/gomail/message.go @@ -0,0 +1,322 @@ +package gomail + +import ( + "bytes" + "io" + "os" + "path/filepath" + "time" +) + +// Message represents an email. +type Message struct { + header header + parts []*part + attachments []*file + embedded []*file + charset string + encoding Encoding + hEncoder mimeEncoder + buf bytes.Buffer +} + +type header map[string][]string + +type part struct { + contentType string + copier func(io.Writer) error + encoding Encoding +} + +// NewMessage creates a new message. It uses UTF-8 and quoted-printable encoding +// by default. +func NewMessage(settings ...MessageSetting) *Message { + m := &Message{ + header: make(header), + charset: "UTF-8", + encoding: QuotedPrintable, + } + + m.applySettings(settings) + + if m.encoding == Base64 { + m.hEncoder = bEncoding + } else { + m.hEncoder = qEncoding + } + + return m +} + +// Reset resets the message so it can be reused. The message keeps its previous +// settings so it is in the same state that after a call to NewMessage. +func (m *Message) Reset() { + for k := range m.header { + delete(m.header, k) + } + m.parts = nil + m.attachments = nil + m.embedded = nil +} + +func (m *Message) applySettings(settings []MessageSetting) { + for _, s := range settings { + s(m) + } +} + +// A MessageSetting can be used as an argument in NewMessage to configure an +// email. +type MessageSetting func(m *Message) + +// SetCharset is a message setting to set the charset of the email. +func SetCharset(charset string) MessageSetting { + return func(m *Message) { + m.charset = charset + } +} + +// SetEncoding is a message setting to set the encoding of the email. +func SetEncoding(enc Encoding) MessageSetting { + return func(m *Message) { + m.encoding = enc + } +} + +// Encoding represents a MIME encoding scheme like quoted-printable or base64. +type Encoding string + +const ( + // QuotedPrintable represents the quoted-printable encoding as defined in + // RFC 2045. + QuotedPrintable Encoding = "quoted-printable" + // Base64 represents the base64 encoding as defined in RFC 2045. + Base64 Encoding = "base64" + // Unencoded can be used to avoid encoding the body of an email. The headers + // will still be encoded using quoted-printable encoding. + Unencoded Encoding = "8bit" +) + +// SetHeader sets a value to the given header field. +func (m *Message) SetHeader(field string, value ...string) { + m.encodeHeader(value) + m.header[field] = value +} + +func (m *Message) encodeHeader(values []string) { + for i := range values { + values[i] = m.encodeString(values[i]) + } +} + +func (m *Message) encodeString(value string) string { + return m.hEncoder.Encode(m.charset, value) +} + +// SetHeaders sets the message headers. +func (m *Message) SetHeaders(h map[string][]string) { + for k, v := range h { + m.SetHeader(k, v...) + } +} + +// SetAddressHeader sets an address to the given header field. +func (m *Message) SetAddressHeader(field, address, name string) { + m.header[field] = []string{m.FormatAddress(address, name)} +} + +// FormatAddress formats an address and a name as a valid RFC 5322 address. +func (m *Message) FormatAddress(address, name string) string { + if name == "" { + return address + } + + enc := m.encodeString(name) + if enc == name { + m.buf.WriteByte('"') + for i := 0; i < len(name); i++ { + b := name[i] + if b == '\\' || b == '"' { + m.buf.WriteByte('\\') + } + m.buf.WriteByte(b) + } + m.buf.WriteByte('"') + } else if hasSpecials(name) { + m.buf.WriteString(bEncoding.Encode(m.charset, name)) + } else { + m.buf.WriteString(enc) + } + m.buf.WriteString(" <") + m.buf.WriteString(address) + m.buf.WriteByte('>') + + addr := m.buf.String() + m.buf.Reset() + return addr +} + +func hasSpecials(text string) bool { + for i := 0; i < len(text); i++ { + switch c := text[i]; c { + case '(', ')', '<', '>', '[', ']', ':', ';', '@', '\\', ',', '.', '"': + return true + } + } + + return false +} + +// SetDateHeader sets a date to the given header field. +func (m *Message) SetDateHeader(field string, date time.Time) { + m.header[field] = []string{m.FormatDate(date)} +} + +// FormatDate formats a date as a valid RFC 5322 date. +func (m *Message) FormatDate(date time.Time) string { + return date.Format(time.RFC1123Z) +} + +// GetHeader gets a header field. +func (m *Message) GetHeader(field string) []string { + return m.header[field] +} + +// SetBody sets the body of the message. It replaces any content previously set +// by SetBody, AddAlternative or AddAlternativeWriter. +func (m *Message) SetBody(contentType, body string, settings ...PartSetting) { + m.parts = []*part{m.newPart(contentType, newCopier(body), settings)} +} + +// AddAlternative adds an alternative part to the message. +// +// It is commonly used to send HTML emails that default to the plain text +// version for backward compatibility. AddAlternative appends the new part to +// the end of the message. So the plain text part should be added before the +// HTML part. See http://en.wikipedia.org/wiki/MIME#Alternative +func (m *Message) AddAlternative(contentType, body string, settings ...PartSetting) { + m.AddAlternativeWriter(contentType, newCopier(body), settings...) +} + +func newCopier(s string) func(io.Writer) error { + return func(w io.Writer) error { + _, err := io.WriteString(w, s) + return err + } +} + +// AddAlternativeWriter adds an alternative part to the message. It can be +// useful with the text/template or html/template packages. +func (m *Message) AddAlternativeWriter(contentType string, f func(io.Writer) error, settings ...PartSetting) { + m.parts = append(m.parts, m.newPart(contentType, f, settings)) +} + +func (m *Message) newPart(contentType string, f func(io.Writer) error, settings []PartSetting) *part { + p := &part{ + contentType: contentType, + copier: f, + encoding: m.encoding, + } + + for _, s := range settings { + s(p) + } + + return p +} + +// A PartSetting can be used as an argument in Message.SetBody, +// Message.AddAlternative or Message.AddAlternativeWriter to configure the part +// added to a message. +type PartSetting func(*part) + +// SetPartEncoding sets the encoding of the part added to the message. By +// default, parts use the same encoding than the message. +func SetPartEncoding(e Encoding) PartSetting { + return PartSetting(func(p *part) { + p.encoding = e + }) +} + +type file struct { + Name string + Header map[string][]string + CopyFunc func(w io.Writer) error +} + +func (f *file) setHeader(field, value string) { + f.Header[field] = []string{value} +} + +// A FileSetting can be used as an argument in Message.Attach or Message.Embed. +type FileSetting func(*file) + +// SetHeader is a file setting to set the MIME header of the message part that +// contains the file content. +// +// Mandatory headers are automatically added if they are not set when sending +// the email. +func SetHeader(h map[string][]string) FileSetting { + return func(f *file) { + for k, v := range h { + f.Header[k] = v + } + } +} + +// Rename is a file setting to set the name of the attachment if the name is +// different than the filename on disk. +func Rename(name string) FileSetting { + return func(f *file) { + f.Name = name + } +} + +// SetCopyFunc is a file setting to replace the function that runs when the +// message is sent. It should copy the content of the file to the io.Writer. +// +// The default copy function opens the file with the given filename, and copy +// its content to the io.Writer. +func SetCopyFunc(f func(io.Writer) error) FileSetting { + return func(fi *file) { + fi.CopyFunc = f + } +} + +func (m *Message) appendFile(list []*file, name string, settings []FileSetting) []*file { + f := &file{ + Name: filepath.Base(name), + Header: make(map[string][]string), + CopyFunc: func(w io.Writer) error { + h, err := os.Open(name) + if err != nil { + return err + } + if _, err := io.Copy(w, h); err != nil { + h.Close() + return err + } + return h.Close() + }, + } + + for _, s := range settings { + s(f) + } + + if list == nil { + return []*file{f} + } + + return append(list, f) +} + +// Attach attaches the files to the email. +func (m *Message) Attach(filename string, settings ...FileSetting) { + m.attachments = m.appendFile(m.attachments, filename, settings) +} + +// Embed embeds the images to the email. +func (m *Message) Embed(filename string, settings ...FileSetting) { + m.embedded = m.appendFile(m.embedded, filename, settings) +} diff --git a/lib/gomail/message_test.go b/lib/gomail/message_test.go new file mode 100644 index 000000000..acceff2a6 --- /dev/null +++ b/lib/gomail/message_test.go @@ -0,0 +1,745 @@ +package gomail + +import ( + "bytes" + "encoding/base64" + "io" + "io/ioutil" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + "time" +) + +func init() { + now = func() time.Time { + return time.Date(2014, 06, 25, 17, 46, 0, 0, time.UTC) + } +} + +type message struct { + from string + to []string + content string +} + +func TestMessage(t *testing.T) { + m := NewMessage() + m.SetAddressHeader("From", "from@example.com", "Señor From") + m.SetHeader("To", m.FormatAddress("to@example.com", "Señor To"), "tobis@example.com") + m.SetAddressHeader("Cc", "cc@example.com", "A, B") + m.SetAddressHeader("X-To", "ccbis@example.com", "à, b") + m.SetDateHeader("X-Date", now()) + m.SetHeader("X-Date-2", m.FormatDate(now())) + m.SetHeader("Subject", "¡Hola, señor!") + m.SetHeaders(map[string][]string{ + "X-Headers": {"Test", "Café"}, + }) + m.SetBody("text/plain", "¡Hola, señor!") + + want := &message{ + from: "from@example.com", + to: []string{ + "to@example.com", + "tobis@example.com", + "cc@example.com", + }, + content: "From: =?UTF-8?q?Se=C3=B1or_From?= \r\n" + + "To: =?UTF-8?q?Se=C3=B1or_To?= , tobis@example.com\r\n" + + "Cc: \"A, B\" \r\n" + + "X-To: =?UTF-8?b?w6AsIGI=?= \r\n" + + "X-Date: Wed, 25 Jun 2014 17:46:00 +0000\r\n" + + "X-Date-2: Wed, 25 Jun 2014 17:46:00 +0000\r\n" + + "X-Headers: Test, =?UTF-8?q?Caf=C3=A9?=\r\n" + + "Subject: =?UTF-8?q?=C2=A1Hola,_se=C3=B1or!?=\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "=C2=A1Hola, se=C3=B1or!", + } + + testMessage(t, m, 0, want) +} + +func TestCustomMessage(t *testing.T) { + m := NewMessage(SetCharset("ISO-8859-1"), SetEncoding(Base64)) + m.SetHeaders(map[string][]string{ + "From": {"from@example.com"}, + "To": {"to@example.com"}, + "Subject": {"Café"}, + }) + m.SetBody("text/html", "¡Hola, señor!") + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Subject: =?ISO-8859-1?b?Q2Fmw6k=?=\r\n" + + "Content-Type: text/html; charset=ISO-8859-1\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + "wqFIb2xhLCBzZcOxb3Ih", + } + + testMessage(t, m, 0, want) +} + +func TestUnencodedMessage(t *testing.T) { + m := NewMessage(SetEncoding(Unencoded)) + m.SetHeaders(map[string][]string{ + "From": {"from@example.com"}, + "To": {"to@example.com"}, + "Subject": {"Café"}, + }) + m.SetBody("text/html", "¡Hola, señor!") + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Subject: =?UTF-8?q?Caf=C3=A9?=\r\n" + + "Content-Type: text/html; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: 8bit\r\n" + + "\r\n" + + "¡Hola, señor!", + } + + testMessage(t, m, 0, want) +} + +func TestRecipients(t *testing.T) { + m := NewMessage() + m.SetHeaders(map[string][]string{ + "From": {"from@example.com"}, + "To": {"to@example.com"}, + "Cc": {"cc@example.com"}, + "Bcc": {"bcc1@example.com", "bcc2@example.com"}, + "Subject": {"Hello!"}, + }) + m.SetBody("text/plain", "Test message") + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com", "cc@example.com", "bcc1@example.com", "bcc2@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Cc: cc@example.com\r\n" + + "Subject: Hello!\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "Test message", + } + + testMessage(t, m, 0, want) +} + +func TestAlternative(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetBody("text/plain", "¡Hola, señor!") + m.AddAlternative("text/html", "¡Hola, señor!") + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: multipart/alternative;\r\n" + + " boundary=_BOUNDARY_1_\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "=C2=A1Hola, se=C3=B1or!\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/html; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "=C2=A1Hola, se=C3=B1or!\r\n" + + "--_BOUNDARY_1_--\r\n", + } + + testMessage(t, m, 1, want) +} + +func TestPartSetting(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetBody("text/plain; format=flowed", "¡Hola, señor!", SetPartEncoding(Unencoded)) + m.AddAlternative("text/html", "¡Hola, señor!") + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: multipart/alternative;\r\n" + + " boundary=_BOUNDARY_1_\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/plain; format=flowed; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: 8bit\r\n" + + "\r\n" + + "¡Hola, señor!\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/html; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "=C2=A1Hola, se=C3=B1or!\r\n" + + "--_BOUNDARY_1_--\r\n", + } + + testMessage(t, m, 1, want) +} + +func TestBodyWriter(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.AddAlternativeWriter("text/plain", func(w io.Writer) error { + _, err := w.Write([]byte("Test message")) + return err + }) + m.AddAlternativeWriter("text/html", func(w io.Writer) error { + _, err := w.Write([]byte("Test HTML")) + return err + }) + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: multipart/alternative;\r\n" + + " boundary=_BOUNDARY_1_\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "Test message\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/html; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "Test HTML\r\n" + + "--_BOUNDARY_1_--\r\n", + } + + testMessage(t, m, 1, want) +} + +func TestAttachmentOnly(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.Attach(mockCopyFile("/tmp/test.pdf")) + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: application/pdf; name=\"test.pdf\"\r\n" + + "Content-Disposition: attachment; filename=\"test.pdf\"\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of test.pdf")), + } + + testMessage(t, m, 0, want) +} + +func TestAttachment(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetBody("text/plain", "Test") + m.Attach(mockCopyFile("/tmp/test.pdf")) + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: multipart/mixed;\r\n" + + " boundary=_BOUNDARY_1_\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "Test\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: application/pdf; name=\"test.pdf\"\r\n" + + "Content-Disposition: attachment; filename=\"test.pdf\"\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of test.pdf")) + "\r\n" + + "--_BOUNDARY_1_--\r\n", + } + + testMessage(t, m, 1, want) +} + +func TestRename(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetBody("text/plain", "Test") + name, copy := mockCopyFile("/tmp/test.pdf") + rename := Rename("another.pdf") + m.Attach(name, copy, rename) + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: multipart/mixed;\r\n" + + " boundary=_BOUNDARY_1_\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "Test\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: application/pdf; name=\"another.pdf\"\r\n" + + "Content-Disposition: attachment; filename=\"another.pdf\"\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of test.pdf")) + "\r\n" + + "--_BOUNDARY_1_--\r\n", + } + + testMessage(t, m, 1, want) +} + +func TestAttachmentsOnly(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.Attach(mockCopyFile("/tmp/test.pdf")) + m.Attach(mockCopyFile("/tmp/test.zip")) + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: multipart/mixed;\r\n" + + " boundary=_BOUNDARY_1_\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: application/pdf; name=\"test.pdf\"\r\n" + + "Content-Disposition: attachment; filename=\"test.pdf\"\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of test.pdf")) + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: application/zip; name=\"test.zip\"\r\n" + + "Content-Disposition: attachment; filename=\"test.zip\"\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of test.zip")) + "\r\n" + + "--_BOUNDARY_1_--\r\n", + } + + testMessage(t, m, 1, want) +} + +func TestAttachments(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetBody("text/plain", "Test") + m.Attach(mockCopyFile("/tmp/test.pdf")) + m.Attach(mockCopyFile("/tmp/test.zip")) + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: multipart/mixed;\r\n" + + " boundary=_BOUNDARY_1_\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "Test\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: application/pdf; name=\"test.pdf\"\r\n" + + "Content-Disposition: attachment; filename=\"test.pdf\"\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of test.pdf")) + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: application/zip; name=\"test.zip\"\r\n" + + "Content-Disposition: attachment; filename=\"test.zip\"\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of test.zip")) + "\r\n" + + "--_BOUNDARY_1_--\r\n", + } + + testMessage(t, m, 1, want) +} + +func TestEmbedded(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.Embed(mockCopyFileWithHeader(m, "image1.jpg", map[string][]string{"Content-ID": {""}})) + m.Embed(mockCopyFile("image2.jpg")) + m.SetBody("text/plain", "Test") + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: multipart/related;\r\n" + + " boundary=_BOUNDARY_1_\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "Test\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: image/jpeg; name=\"image1.jpg\"\r\n" + + "Content-Disposition: inline; filename=\"image1.jpg\"\r\n" + + "Content-ID: \r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of image1.jpg")) + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: image/jpeg; name=\"image2.jpg\"\r\n" + + "Content-Disposition: inline; filename=\"image2.jpg\"\r\n" + + "Content-ID: \r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of image2.jpg")) + "\r\n" + + "--_BOUNDARY_1_--\r\n", + } + + testMessage(t, m, 1, want) +} + +func TestFullMessage(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetBody("text/plain", "¡Hola, señor!") + m.AddAlternative("text/html", "¡Hola, señor!") + m.Attach(mockCopyFile("test.pdf")) + m.Embed(mockCopyFile("image.jpg")) + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: multipart/mixed;\r\n" + + " boundary=_BOUNDARY_1_\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: multipart/related;\r\n" + + " boundary=_BOUNDARY_2_\r\n" + + "\r\n" + + "--_BOUNDARY_2_\r\n" + + "Content-Type: multipart/alternative;\r\n" + + " boundary=_BOUNDARY_3_\r\n" + + "\r\n" + + "--_BOUNDARY_3_\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "=C2=A1Hola, se=C3=B1or!\r\n" + + "--_BOUNDARY_3_\r\n" + + "Content-Type: text/html; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "=C2=A1Hola, se=C3=B1or!\r\n" + + "--_BOUNDARY_3_--\r\n" + + "\r\n" + + "--_BOUNDARY_2_\r\n" + + "Content-Type: image/jpeg; name=\"image.jpg\"\r\n" + + "Content-Disposition: inline; filename=\"image.jpg\"\r\n" + + "Content-ID: \r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of image.jpg")) + "\r\n" + + "--_BOUNDARY_2_--\r\n" + + "\r\n" + + "--_BOUNDARY_1_\r\n" + + "Content-Type: application/pdf; name=\"test.pdf\"\r\n" + + "Content-Disposition: attachment; filename=\"test.pdf\"\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + base64.StdEncoding.EncodeToString([]byte("Content of test.pdf")) + "\r\n" + + "--_BOUNDARY_1_--\r\n", + } + + testMessage(t, m, 3, want) + + want = &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "Test reset", + } + m.Reset() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetBody("text/plain", "Test reset") + testMessage(t, m, 0, want) +} + +func TestQpLineLength(t *testing.T) { + m := NewMessage() + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetBody("text/plain", + strings.Repeat("0", 76)+"\r\n"+ + strings.Repeat("0", 75)+"à\r\n"+ + strings.Repeat("0", 74)+"à\r\n"+ + strings.Repeat("0", 73)+"à\r\n"+ + strings.Repeat("0", 72)+"à\r\n"+ + strings.Repeat("0", 75)+"\r\n"+ + strings.Repeat("0", 76)+"\n") + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + strings.Repeat("0", 75) + "=\r\n0\r\n" + + strings.Repeat("0", 75) + "=\r\n=C3=A0\r\n" + + strings.Repeat("0", 74) + "=\r\n=C3=A0\r\n" + + strings.Repeat("0", 73) + "=\r\n=C3=A0\r\n" + + strings.Repeat("0", 72) + "=C3=\r\n=A0\r\n" + + strings.Repeat("0", 75) + "\r\n" + + strings.Repeat("0", 75) + "=\r\n0\r\n", + } + + testMessage(t, m, 0, want) +} + +func TestBase64LineLength(t *testing.T) { + m := NewMessage(SetCharset("UTF-8"), SetEncoding(Base64)) + m.SetHeader("From", "from@example.com") + m.SetHeader("To", "to@example.com") + m.SetBody("text/plain", strings.Repeat("0", 58)) + + want := &message{ + from: "from@example.com", + to: []string{"to@example.com"}, + content: "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: base64\r\n" + + "\r\n" + + strings.Repeat("MDAw", 19) + "\r\nMA==", + } + + testMessage(t, m, 0, want) +} + +func TestEmptyName(t *testing.T) { + m := NewMessage() + m.SetAddressHeader("From", "from@example.com", "") + + want := &message{ + from: "from@example.com", + content: "From: from@example.com\r\n", + } + + testMessage(t, m, 0, want) +} + +func TestEmptyHeader(t *testing.T) { + m := NewMessage() + m.SetHeaders(map[string][]string{ + "From": {"from@example.com"}, + "X-Empty": nil, + }) + + want := &message{ + from: "from@example.com", + content: "From: from@example.com\r\n" + + "X-Empty:\r\n", + } + + testMessage(t, m, 0, want) +} + +func testMessage(t *testing.T, m *Message, bCount int, want *message) { + err := Send(stubSendMail(t, bCount, want), m) + if err != nil { + t.Error(err) + } +} + +func stubSendMail(t *testing.T, bCount int, want *message) SendFunc { + return func(from string, to []string, m io.WriterTo) error { + if from != want.from { + t.Fatalf("Invalid from, got %q, want %q", from, want.from) + } + + if len(to) != len(want.to) { + t.Fatalf("Invalid recipient count, \ngot %d: %q\nwant %d: %q", + len(to), to, + len(want.to), want.to, + ) + } + for i := range want.to { + if to[i] != want.to[i] { + t.Fatalf("Invalid recipient, got %q, want %q", + to[i], want.to[i], + ) + } + } + + buf := new(bytes.Buffer) + _, err := m.WriteTo(buf) + if err != nil { + t.Error(err) + } + got := buf.String() + wantMsg := string("Mime-Version: 1.0\r\n" + + "Date: Wed, 25 Jun 2014 17:46:00 +0000\r\n" + + want.content) + if bCount > 0 { + boundaries := getBoundaries(t, bCount, got) + for i, b := range boundaries { + wantMsg = strings.Replace(wantMsg, "_BOUNDARY_"+strconv.Itoa(i+1)+"_", b, -1) + } + } + + compareBodies(t, got, wantMsg) + + return nil + } +} + +func compareBodies(t *testing.T, got, want string) { + // We cannot do a simple comparison since the ordering of headers' fields + // is random. + gotLines := strings.Split(got, "\r\n") + wantLines := strings.Split(want, "\r\n") + + // We only test for too many lines, missing lines are tested after + if len(gotLines) > len(wantLines) { + t.Fatalf("Message has too many lines, \ngot %d:\n%s\nwant %d:\n%s", len(gotLines), got, len(wantLines), want) + } + + isInHeader := true + headerStart := 0 + for i, line := range wantLines { + if line == gotLines[i] { + if line == "" { + isInHeader = false + } else if !isInHeader && len(line) > 2 && line[:2] == "--" { + isInHeader = true + headerStart = i + 1 + } + continue + } + + if !isInHeader { + missingLine(t, line, got, want) + } + + isMissing := true + for j := headerStart; j < len(gotLines); j++ { + if gotLines[j] == "" { + break + } + if gotLines[j] == line { + isMissing = false + break + } + } + if isMissing { + missingLine(t, line, got, want) + } + } +} + +func missingLine(t *testing.T, line, got, want string) { + t.Fatalf("Missing line %q\ngot:\n%s\nwant:\n%s", line, got, want) +} + +func getBoundaries(t *testing.T, count int, m string) []string { + if matches := boundaryRegExp.FindAllStringSubmatch(m, count); matches != nil { + boundaries := make([]string, count) + for i, match := range matches { + boundaries[i] = match[1] + } + return boundaries + } + + t.Fatal("Boundary not found in body") + return []string{""} +} + +var boundaryRegExp = regexp.MustCompile("boundary=(\\w+)") + +func mockCopyFile(name string) (string, FileSetting) { + return name, SetCopyFunc(func(w io.Writer) error { + _, err := w.Write([]byte("Content of " + filepath.Base(name))) + return err + }) +} + +func mockCopyFileWithHeader(m *Message, name string, h map[string][]string) (string, FileSetting, FileSetting) { + name, f := mockCopyFile(name) + return name, f, SetHeader(h) +} + +func BenchmarkFull(b *testing.B) { + discardFunc := SendFunc(func(from string, to []string, m io.WriterTo) error { + _, err := m.WriteTo(ioutil.Discard) + return err + }) + + m := NewMessage() + b.ResetTimer() + for n := 0; n < b.N; n++ { + m.SetAddressHeader("From", "from@example.com", "Señor From") + m.SetHeaders(map[string][]string{ + "To": {"to@example.com"}, + "Cc": {"cc@example.com"}, + "Bcc": {"bcc1@example.com", "bcc2@example.com"}, + "Subject": {"¡Hola, señor!"}, + }) + m.SetBody("text/plain", "¡Hola, señor!") + m.AddAlternative("text/html", "

¡Hola, señor!

") + m.Attach(mockCopyFile("benchmark.txt")) + m.Embed(mockCopyFile("benchmark.jpg")) + + if err := Send(discardFunc, m); err != nil { + panic(err) + } + m.Reset() + } +} diff --git a/lib/gomail/mime.go b/lib/gomail/mime.go new file mode 100644 index 000000000..26c44def3 --- /dev/null +++ b/lib/gomail/mime.go @@ -0,0 +1,22 @@ +//go:build go1.5 +// +build go1.5 + +package gomail + +import ( + "mime" + "mime/quotedprintable" + "strings" +) + +var newQPWriter = quotedprintable.NewWriter + +type mimeEncoder struct { + mime.WordEncoder +} + +var ( + bEncoding = mimeEncoder{mime.BEncoding} + qEncoding = mimeEncoder{mime.QEncoding} + lastIndexByte = strings.LastIndexByte +) diff --git a/lib/gomail/mime_go14.go b/lib/gomail/mime_go14.go new file mode 100644 index 000000000..6255dbc42 --- /dev/null +++ b/lib/gomail/mime_go14.go @@ -0,0 +1,26 @@ +//go:build !go1.5 +// +build !go1.5 + +package gomail + +import "gopkg.in/alexcesaro/quotedprintable.v3" + +var newQPWriter = quotedprintable.NewWriter + +type mimeEncoder struct { + quotedprintable.WordEncoder +} + +var ( + bEncoding = mimeEncoder{quotedprintable.BEncoding} + qEncoding = mimeEncoder{quotedprintable.QEncoding} + lastIndexByte = func(s string, c byte) int { + for i := len(s) - 1; i >= 0; i-- { + + if s[i] == c { + return i + } + } + return -1 + } +) diff --git a/lib/gomail/send.go b/lib/gomail/send.go new file mode 100644 index 000000000..9115ebe72 --- /dev/null +++ b/lib/gomail/send.go @@ -0,0 +1,116 @@ +package gomail + +import ( + "errors" + "fmt" + "io" + "net/mail" +) + +// Sender is the interface that wraps the Send method. +// +// Send sends an email to the given addresses. +type Sender interface { + Send(from string, to []string, msg io.WriterTo) error +} + +// SendCloser is the interface that groups the Send and Close methods. +type SendCloser interface { + Sender + Close() error +} + +// A SendFunc is a function that sends emails to the given addresses. +// +// The SendFunc type is an adapter to allow the use of ordinary functions as +// email senders. If f is a function with the appropriate signature, SendFunc(f) +// is a Sender object that calls f. +type SendFunc func(from string, to []string, msg io.WriterTo) error + +// Send calls f(from, to, msg). +func (f SendFunc) Send(from string, to []string, msg io.WriterTo) error { + return f(from, to, msg) +} + +// Send sends emails using the given Sender. +func Send(s Sender, msg ...*Message) error { + for i, m := range msg { + if err := send(s, m); err != nil { + return fmt.Errorf("gomail: could not send email %d: %v", i+1, err) + } + } + + return nil +} + +func send(s Sender, m *Message) error { + from, err := m.getFrom() + if err != nil { + return err + } + + to, err := m.getRecipients() + if err != nil { + return err + } + + if err := s.Send(from, to, m); err != nil { + return err + } + + return nil +} + +func (m *Message) getFrom() (string, error) { + from := m.header["Sender"] + if len(from) == 0 { + from = m.header["From"] + if len(from) == 0 { + return "", errors.New(`gomail: invalid message, "From" field is absent`) + } + } + + return parseAddress(from[0]) +} + +func (m *Message) getRecipients() ([]string, error) { + n := 0 + for _, field := range []string{"To", "Cc", "Bcc"} { + if addresses, ok := m.header[field]; ok { + n += len(addresses) + } + } + list := make([]string, 0, n) + + for _, field := range []string{"To", "Cc", "Bcc"} { + if addresses, ok := m.header[field]; ok { + for _, a := range addresses { + addr, err := parseAddress(a) + if err != nil { + return nil, err + } + list = addAddress(list, addr) + } + } + } + + return list, nil +} + +func addAddress(list []string, addr string) []string { + for _, a := range list { + if addr == a { + return list + } + } + + return append(list, addr) +} + +func parseAddress(field string) (string, error) { + addr, err := mail.ParseAddress(field) + if err != nil { + return "", fmt.Errorf("gomail: invalid address %q: %v", field, err) + } + return addr.Address, nil +} diff --git a/lib/gomail/send_test.go b/lib/gomail/send_test.go new file mode 100644 index 000000000..ba59cd3dc --- /dev/null +++ b/lib/gomail/send_test.go @@ -0,0 +1,80 @@ +package gomail + +import ( + "bytes" + "io" + "reflect" + "testing" +) + +const ( + testTo1 = "to1@example.com" + testTo2 = "to2@example.com" + testFrom = "from@example.com" + testBody = "Test message" + testMsg = "To: " + testTo1 + ", " + testTo2 + "\r\n" + + "From: " + testFrom + "\r\n" + + "Mime-Version: 1.0\r\n" + + "Date: Wed, 25 Jun 2014 17:46:00 +0000\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + testBody +) + +type mockSender SendFunc + +func (s mockSender) Send(from string, to []string, msg io.WriterTo) error { + return s(from, to, msg) +} + +type mockSendCloser struct { + mockSender + close func() error +} + +func (s *mockSendCloser) Close() error { + return s.close() +} + +func TestSend(t *testing.T) { + s := &mockSendCloser{ + mockSender: stubSend(t, testFrom, []string{testTo1, testTo2}, testMsg), + close: func() error { + t.Error("Close() should not be called in Send()") + return nil + }, + } + if err := Send(s, getTestMessage()); err != nil { + t.Errorf("Send(): %v", err) + } +} + +func getTestMessage() *Message { + m := NewMessage() + m.SetHeader("From", testFrom) + m.SetHeader("To", testTo1, testTo2) + m.SetBody("text/plain", testBody) + + return m +} + +func stubSend(t *testing.T, wantFrom string, wantTo []string, wantBody string) mockSender { + return func(from string, to []string, msg io.WriterTo) error { + if from != wantFrom { + t.Errorf("invalid from, got %q, want %q", from, wantFrom) + } + if !reflect.DeepEqual(to, wantTo) { + t.Errorf("invalid to, got %v, want %v", to, wantTo) + } + + buf := new(bytes.Buffer) + _, err := msg.WriteTo(buf) + if err != nil { + t.Fatal(err) + } + compareBodies(t, buf.String(), wantBody) + + return nil + } +} diff --git a/lib/gomail/smtp.go b/lib/gomail/smtp.go new file mode 100644 index 000000000..024c3adb7 --- /dev/null +++ b/lib/gomail/smtp.go @@ -0,0 +1,209 @@ +package gomail + +import ( + "crypto/tls" + "fmt" + "io" + "net" + "net/smtp" + "strings" + "time" +) + +// A Dialer is a dialer to an SMTP server. +type Dialer struct { + // Host represents the host of the SMTP server. + Host string + // Port represents the port of the SMTP server. + Port int + Timeout time.Duration + // Username is the username to use to authenticate to the SMTP server. + Username string + // Password is the password to use to authenticate to the SMTP server. + Password string + // Auth represents the authentication mechanism used to authenticate to the + // SMTP server. + Auth smtp.Auth + // SSL defines whether an SSL connection is used. It should be false in + // most cases since the authentication mechanism should use the STARTTLS + // extension instead. + SSL bool + // TSLConfig represents the TLS configuration used for the TLS (when the + // STARTTLS extension is used) or SSL connection. + TLSConfig *tls.Config + // LocalName is the hostname sent to the SMTP server with the HELO command. + // By default, "localhost" is sent. + LocalName string +} + +// NewDialer returns a new SMTP Dialer. The given parameters are used to connect +// to the SMTP server. +func NewDialer(host string, port int, username, password string) *Dialer { + return NewDialerWithTimeout(host, port, username, password, 30*time.Second) +} + +func NewDialerWithTimeout(host string, port int, username, password string, timeout time.Duration) *Dialer { + return &Dialer{ + Host: host, + Port: port, + Username: username, + Password: password, + SSL: port == 465, + Timeout: timeout, + } +} + +// NewPlainDialer returns a new SMTP Dialer. The given parameters are used to +// connect to the SMTP server. +// +// Deprecated: Use NewDialer instead. +func NewPlainDialer(host string, port int, username, password string) *Dialer { + return NewDialer(host, port, username, password) +} + +// Dial dials and authenticates to an SMTP server. The returned SendCloser +// should be closed when done using it. +func (d *Dialer) Dial() (SendCloser, error) { + conn, err := netDialTimeout("tcp", addr(d.Host, d.Port), d.Timeout) + if err != nil { + return nil, err + } + + if d.SSL { + conn = tlsClient(conn, d.tlsConfig()) + } + + c, err := smtpNewClient(conn, d.Host) + if err != nil { + return nil, err + } + + if d.LocalName != "" { + if err := c.Hello(d.LocalName); err != nil { + return nil, err + } + } + + if !d.SSL { + if ok, _ := c.Extension("STARTTLS"); ok { + if err := c.StartTLS(d.tlsConfig()); err != nil { + c.Close() + return nil, err + } + } + } + + if d.Auth == nil && d.Username != "" { + if ok, auths := c.Extension("AUTH"); ok { + if strings.Contains(auths, "CRAM-MD5") { + d.Auth = smtp.CRAMMD5Auth(d.Username, d.Password) + } else if strings.Contains(auths, "LOGIN") && + !strings.Contains(auths, "PLAIN") { + d.Auth = &loginAuth{ + username: d.Username, + password: d.Password, + host: d.Host, + } + } else { + d.Auth = smtp.PlainAuth("", d.Username, d.Password, d.Host) + } + } + } + + if d.Auth != nil { + if err = c.Auth(d.Auth); err != nil { + c.Close() + return nil, err + } + } + + return &smtpSender{c, d}, nil +} + +func (d *Dialer) tlsConfig() *tls.Config { + if d.TLSConfig == nil { + return &tls.Config{ServerName: d.Host} + } + return d.TLSConfig +} + +func addr(host string, port int) string { + return fmt.Sprintf("%s:%d", host, port) +} + +// DialAndSend opens a connection to the SMTP server, sends the given emails and +// closes the connection. +func (d *Dialer) DialAndSend(m ...*Message) error { + s, err := d.Dial() + if err != nil { + panic(err) + return err + } + defer s.Close() + + return Send(s, m...) +} + +type smtpSender struct { + smtpClient + d *Dialer +} + +func (c *smtpSender) Send(from string, to []string, msg io.WriterTo) error { + if err := c.Mail(from); err != nil { + if err == io.EOF { + // This is probably due to a timeout, so reconnect and try again. + sc, derr := c.d.Dial() + if derr == nil { + if s, ok := sc.(*smtpSender); ok { + *c = *s + return c.Send(from, to, msg) + } + } + } + return err + } + + for _, addr := range to { + if err := c.Rcpt(addr); err != nil { + return err + } + } + + w, err := c.Data() + if err != nil { + return err + } + + if _, err = msg.WriteTo(w); err != nil { + w.Close() + return err + } + + return w.Close() +} + +func (c *smtpSender) Close() error { + return c.Quit() +} + +// Stubbed out for tests. +var ( + netDialTimeout = net.DialTimeout + tlsClient = tls.Client + smtpNewClient = func(conn net.Conn, host string) (smtpClient, error) { + return smtp.NewClient(conn, host) + } +) + +type smtpClient interface { + Hello(string) error + Extension(string) (bool, string) + StartTLS(*tls.Config) error + Auth(smtp.Auth) error + Mail(string) error + Rcpt(string) error + Data() (io.WriteCloser, error) + Quit() error + Close() error +} diff --git a/lib/gomail/smtp_test.go b/lib/gomail/smtp_test.go new file mode 100644 index 000000000..b6f91555b --- /dev/null +++ b/lib/gomail/smtp_test.go @@ -0,0 +1,292 @@ +package gomail + +import ( + "bytes" + "crypto/tls" + "io" + "net" + "net/smtp" + "reflect" + "testing" + "time" +) + +const ( + testPort = 587 + testSSLPort = 465 +) + +var ( + testConn = &net.TCPConn{} + testTLSConn = &tls.Conn{} + testConfig = &tls.Config{InsecureSkipVerify: true} + testAuth = smtp.PlainAuth("", testUser, testPwd, testHost) +) + +func TestDialer(t *testing.T) { + d := NewDialer(testHost, testPort, "user", "pwd") + testSendMail(t, d, []string{ + "Extension STARTTLS", + "StartTLS", + "Extension AUTH", + "Auth", + "Mail " + testFrom, + "Rcpt " + testTo1, + "Rcpt " + testTo2, + "Data", + "Write message", + "Close writer", + "Quit", + "Close", + }) +} + +func TestDialerSSL(t *testing.T) { + d := NewDialer(testHost, testSSLPort, "user", "pwd") + testSendMail(t, d, []string{ + "Extension AUTH", + "Auth", + "Mail " + testFrom, + "Rcpt " + testTo1, + "Rcpt " + testTo2, + "Data", + "Write message", + "Close writer", + "Quit", + "Close", + }) +} + +func TestDialerConfig(t *testing.T) { + d := NewDialer(testHost, testPort, "user", "pwd") + d.LocalName = "test" + d.TLSConfig = testConfig + testSendMail(t, d, []string{ + "Hello test", + "Extension STARTTLS", + "StartTLS", + "Extension AUTH", + "Auth", + "Mail " + testFrom, + "Rcpt " + testTo1, + "Rcpt " + testTo2, + "Data", + "Write message", + "Close writer", + "Quit", + "Close", + }) +} + +func TestDialerSSLConfig(t *testing.T) { + d := NewDialer(testHost, testSSLPort, "user", "pwd") + d.LocalName = "test" + d.TLSConfig = testConfig + testSendMail(t, d, []string{ + "Hello test", + "Extension AUTH", + "Auth", + "Mail " + testFrom, + "Rcpt " + testTo1, + "Rcpt " + testTo2, + "Data", + "Write message", + "Close writer", + "Quit", + "Close", + }) +} + +func TestDialerNoAuth(t *testing.T) { + d := &Dialer{ + Host: testHost, + Port: testPort, + } + testSendMail(t, d, []string{ + "Extension STARTTLS", + "StartTLS", + "Mail " + testFrom, + "Rcpt " + testTo1, + "Rcpt " + testTo2, + "Data", + "Write message", + "Close writer", + "Quit", + "Close", + }) +} + +func TestDialerTimeout(t *testing.T) { + d := &Dialer{ + Host: testHost, + Port: testPort, + } + testSendMailTimeout(t, d, []string{ + "Extension STARTTLS", + "StartTLS", + "Mail " + testFrom, + "Extension STARTTLS", + "StartTLS", + "Mail " + testFrom, + "Rcpt " + testTo1, + "Rcpt " + testTo2, + "Data", + "Write message", + "Close writer", + "Quit", + "Close", + }) +} + +type mockClient struct { + t *testing.T + i int + want []string + addr string + config *tls.Config + timeout bool +} + +func (c *mockClient) Hello(localName string) error { + c.do("Hello " + localName) + return nil +} + +func (c *mockClient) Extension(ext string) (bool, string) { + c.do("Extension " + ext) + return true, "" +} + +func (c *mockClient) StartTLS(config *tls.Config) error { + assertConfig(c.t, config, c.config) + c.do("StartTLS") + return nil +} + +func (c *mockClient) Auth(a smtp.Auth) error { + if !reflect.DeepEqual(a, testAuth) { + c.t.Errorf("Invalid auth, got %#v, want %#v", a, testAuth) + } + c.do("Auth") + return nil +} + +func (c *mockClient) Mail(from string) error { + c.do("Mail " + from) + if c.timeout { + c.timeout = false + return io.EOF + } + return nil +} + +func (c *mockClient) Rcpt(to string) error { + c.do("Rcpt " + to) + return nil +} + +func (c *mockClient) Data() (io.WriteCloser, error) { + c.do("Data") + return &mockWriter{c: c, want: testMsg}, nil +} + +func (c *mockClient) Quit() error { + c.do("Quit") + return nil +} + +func (c *mockClient) Close() error { + c.do("Close") + return nil +} + +func (c *mockClient) do(cmd string) { + if c.i >= len(c.want) { + c.t.Fatalf("Invalid command %q", cmd) + } + + if cmd != c.want[c.i] { + c.t.Fatalf("Invalid command, got %q, want %q", cmd, c.want[c.i]) + } + c.i++ +} + +type mockWriter struct { + want string + c *mockClient + buf bytes.Buffer +} + +func (w *mockWriter) Write(p []byte) (int, error) { + if w.buf.Len() == 0 { + w.c.do("Write message") + } + w.buf.Write(p) + return len(p), nil +} + +func (w *mockWriter) Close() error { + compareBodies(w.c.t, w.buf.String(), w.want) + w.c.do("Close writer") + return nil +} + +func testSendMail(t *testing.T, d *Dialer, want []string) { + doTestSendMail(t, d, want, false) +} + +func testSendMailTimeout(t *testing.T, d *Dialer, want []string) { + doTestSendMail(t, d, want, true) +} + +func doTestSendMail(t *testing.T, d *Dialer, want []string, timeout bool) { + testClient := &mockClient{ + t: t, + want: want, + addr: addr(d.Host, d.Port), + config: d.TLSConfig, + timeout: timeout, + } + + netDialTimeout = func(network, address string, d time.Duration) (net.Conn, error) { + if network != "tcp" { + t.Errorf("Invalid network, got %q, want tcp", network) + } + if address != testClient.addr { + t.Errorf("Invalid address, got %q, want %q", + address, testClient.addr) + } + return testConn, nil + } + + tlsClient = func(conn net.Conn, config *tls.Config) *tls.Conn { + if conn != testConn { + t.Errorf("Invalid conn, got %#v, want %#v", conn, testConn) + } + assertConfig(t, config, testClient.config) + return testTLSConn + } + + smtpNewClient = func(conn net.Conn, host string) (smtpClient, error) { + if host != testHost { + t.Errorf("Invalid host, got %q, want %q", host, testHost) + } + return testClient, nil + } + + if err := d.DialAndSend(getTestMessage()); err != nil { + t.Error(err) + } +} + +func assertConfig(t *testing.T, got, want *tls.Config) { + if want == nil { + want = &tls.Config{ServerName: testHost} + } + if got.ServerName != want.ServerName { + t.Errorf("Invalid field ServerName in config, got %q, want %q", got.ServerName, want.ServerName) + } + if got.InsecureSkipVerify != want.InsecureSkipVerify { + t.Errorf("Invalid field InsecureSkipVerify in config, got %v, want %v", got.InsecureSkipVerify, want.InsecureSkipVerify) + } +} diff --git a/lib/gomail/writeto.go b/lib/gomail/writeto.go new file mode 100644 index 000000000..9fb6b86e8 --- /dev/null +++ b/lib/gomail/writeto.go @@ -0,0 +1,306 @@ +package gomail + +import ( + "encoding/base64" + "errors" + "io" + "mime" + "mime/multipart" + "path/filepath" + "strings" + "time" +) + +// WriteTo implements io.WriterTo. It dumps the whole message into w. +func (m *Message) WriteTo(w io.Writer) (int64, error) { + mw := &messageWriter{w: w} + mw.writeMessage(m) + return mw.n, mw.err +} + +func (w *messageWriter) writeMessage(m *Message) { + if _, ok := m.header["Mime-Version"]; !ok { + w.writeString("Mime-Version: 1.0\r\n") + } + if _, ok := m.header["Date"]; !ok { + w.writeHeader("Date", m.FormatDate(now())) + } + w.writeHeaders(m.header) + + if m.hasMixedPart() { + w.openMultipart("mixed") + } + + if m.hasRelatedPart() { + w.openMultipart("related") + } + + if m.hasAlternativePart() { + w.openMultipart("alternative") + } + for _, part := range m.parts { + w.writePart(part, m.charset) + } + if m.hasAlternativePart() { + w.closeMultipart() + } + + w.addFiles(m.embedded, false) + if m.hasRelatedPart() { + w.closeMultipart() + } + + w.addFiles(m.attachments, true) + if m.hasMixedPart() { + w.closeMultipart() + } +} + +func (m *Message) hasMixedPart() bool { + return (len(m.parts) > 0 && len(m.attachments) > 0) || len(m.attachments) > 1 +} + +func (m *Message) hasRelatedPart() bool { + return (len(m.parts) > 0 && len(m.embedded) > 0) || len(m.embedded) > 1 +} + +func (m *Message) hasAlternativePart() bool { + return len(m.parts) > 1 +} + +type messageWriter struct { + w io.Writer + n int64 + writers [3]*multipart.Writer + partWriter io.Writer + depth uint8 + err error +} + +func (w *messageWriter) openMultipart(mimeType string) { + mw := multipart.NewWriter(w) + contentType := "multipart/" + mimeType + ";\r\n boundary=" + mw.Boundary() + w.writers[w.depth] = mw + + if w.depth == 0 { + w.writeHeader("Content-Type", contentType) + w.writeString("\r\n") + } else { + w.createPart(map[string][]string{ + "Content-Type": {contentType}, + }) + } + w.depth++ +} + +func (w *messageWriter) createPart(h map[string][]string) { + w.partWriter, w.err = w.writers[w.depth-1].CreatePart(h) +} + +func (w *messageWriter) closeMultipart() { + if w.depth > 0 { + w.writers[w.depth-1].Close() + w.depth-- + } +} + +func (w *messageWriter) writePart(p *part, charset string) { + w.writeHeaders(map[string][]string{ + "Content-Type": {p.contentType + "; charset=" + charset}, + "Content-Transfer-Encoding": {string(p.encoding)}, + }) + w.writeBody(p.copier, p.encoding) +} + +func (w *messageWriter) addFiles(files []*file, isAttachment bool) { + for _, f := range files { + if _, ok := f.Header["Content-Type"]; !ok { + mediaType := mime.TypeByExtension(filepath.Ext(f.Name)) + if mediaType == "" { + mediaType = "application/octet-stream" + } + f.setHeader("Content-Type", mediaType+`; name="`+f.Name+`"`) + } + + if _, ok := f.Header["Content-Transfer-Encoding"]; !ok { + f.setHeader("Content-Transfer-Encoding", string(Base64)) + } + + if _, ok := f.Header["Content-Disposition"]; !ok { + var disp string + if isAttachment { + disp = "attachment" + } else { + disp = "inline" + } + f.setHeader("Content-Disposition", disp+`; filename="`+f.Name+`"`) + } + + if !isAttachment { + if _, ok := f.Header["Content-ID"]; !ok { + f.setHeader("Content-ID", "<"+f.Name+">") + } + } + w.writeHeaders(f.Header) + w.writeBody(f.CopyFunc, Base64) + } +} + +func (w *messageWriter) Write(p []byte) (int, error) { + if w.err != nil { + return 0, errors.New("gomail: cannot write as writer is in error") + } + + var n int + n, w.err = w.w.Write(p) + w.n += int64(n) + return n, w.err +} + +func (w *messageWriter) writeString(s string) { + n, _ := io.WriteString(w.w, s) + w.n += int64(n) +} + +func (w *messageWriter) writeHeader(k string, v ...string) { + w.writeString(k) + if len(v) == 0 { + w.writeString(":\r\n") + return + } + w.writeString(": ") + + // Max header line length is 78 characters in RFC 5322 and 76 characters + // in RFC 2047. So for the sake of simplicity we use the 76 characters + // limit. + charsLeft := 76 - len(k) - len(": ") + + for i, s := range v { + // If the line is already too long, insert a newline right away. + if charsLeft < 1 { + if i == 0 { + w.writeString("\r\n ") + } else { + w.writeString(",\r\n ") + } + charsLeft = 75 + } else if i != 0 { + w.writeString(", ") + charsLeft -= 2 + } + + // While the header content is too long, fold it by inserting a newline. + for len(s) > charsLeft { + s = w.writeLine(s, charsLeft) + charsLeft = 75 + } + w.writeString(s) + if i := lastIndexByte(s, '\n'); i != -1 { + charsLeft = 75 - (len(s) - i - 1) + } else { + charsLeft -= len(s) + } + } + w.writeString("\r\n") +} + +func (w *messageWriter) writeLine(s string, charsLeft int) string { + // If there is already a newline before the limit. Write the line. + if i := strings.IndexByte(s, '\n'); i != -1 && i < charsLeft { + w.writeString(s[:i+1]) + return s[i+1:] + } + + for i := charsLeft - 1; i >= 0; i-- { + if s[i] == ' ' { + w.writeString(s[:i]) + w.writeString("\r\n ") + return s[i+1:] + } + } + + // We could not insert a newline cleanly so look for a space or a newline + // even if it is after the limit. + for i := 75; i < len(s); i++ { + if s[i] == ' ' { + w.writeString(s[:i]) + w.writeString("\r\n ") + return s[i+1:] + } + if s[i] == '\n' { + w.writeString(s[:i+1]) + return s[i+1:] + } + } + + // Too bad, no space or newline in the whole string. Just write everything. + w.writeString(s) + return "" +} + +func (w *messageWriter) writeHeaders(h map[string][]string) { + if w.depth == 0 { + for k, v := range h { + if k != "Bcc" { + w.writeHeader(k, v...) + } + } + } else { + w.createPart(h) + } +} + +func (w *messageWriter) writeBody(f func(io.Writer) error, enc Encoding) { + var subWriter io.Writer + if w.depth == 0 { + w.writeString("\r\n") + subWriter = w.w + } else { + subWriter = w.partWriter + } + + if enc == Base64 { + wc := base64.NewEncoder(base64.StdEncoding, newBase64LineWriter(subWriter)) + w.err = f(wc) + wc.Close() + } else if enc == Unencoded { + w.err = f(subWriter) + } else { + wc := newQPWriter(subWriter) + w.err = f(wc) + wc.Close() + } +} + +// As required by RFC 2045, 6.7. (page 21) for quoted-printable, and +// RFC 2045, 6.8. (page 25) for base64. +const maxLineLen = 76 + +// base64LineWriter limits text encoded in base64 to 76 characters per line +type base64LineWriter struct { + w io.Writer + lineLen int +} + +func newBase64LineWriter(w io.Writer) *base64LineWriter { + return &base64LineWriter{w: w} +} + +func (w *base64LineWriter) Write(p []byte) (int, error) { + n := 0 + for len(p)+w.lineLen > maxLineLen { + w.w.Write(p[:maxLineLen-w.lineLen]) + w.w.Write([]byte("\r\n")) + p = p[maxLineLen-w.lineLen:] + n += maxLineLen - w.lineLen + w.lineLen = 0 + } + + w.w.Write(p) + w.lineLen += len(p) + + return n + len(p), nil +} + +// Stubbed out for testing. +var now = time.Now diff --git a/lib/guardian/auth/strategies/ldap/ldap.go b/lib/guardian/auth/strategies/ldap/ldap.go index a3cfb11f6..02ee68a3b 100644 --- a/lib/guardian/auth/strategies/ldap/ldap.go +++ b/lib/guardian/auth/strategies/ldap/ldap.go @@ -7,12 +7,13 @@ import ( "crypto/tls" "errors" "fmt" + "strings" + "github.com/go-ldap/ldap/v3" "infini.sh/framework/core/util" "infini.sh/framework/lib/fasthttp" "infini.sh/framework/lib/guardian/auth" "infini.sh/framework/lib/guardian/auth/strategies/basic" - "strings" ) // ErrEntries is returned by ldap authenticate function, @@ -24,7 +25,7 @@ type conn interface { Search(searchRequest *ldap.SearchRequest) (*ldap.SearchResult, error) StartTLS(config *tls.Config) error UnauthenticatedBind(username string) error - Close() + Close() error } // Config define the configuration to connect to LDAP. diff --git a/lib/guardian/go.sum b/lib/guardian/go.sum index 617929ca1..e432edd2b 100644 --- a/lib/guardian/go.sum +++ b/lib/guardian/go.sum @@ -135,7 +135,6 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm google.golang.org/api v0.0.0-20170921000349-586095a6e407/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= diff --git a/lib/lock_free/queue/esQueue_test.go b/lib/lock_free/queue/esQueue_test.go deleted file mode 100755 index a75bb846b..000000000 --- a/lib/lock_free/queue/esQueue_test.go +++ /dev/null @@ -1,451 +0,0 @@ -// esQueue_test -package queue - -import ( - "fmt" - "os" - "runtime" - "sync" - "sync/atomic" - "testing" - "time" -) - -func TestQueue(t *testing.T) { - q := NewQueue(8) - ok, quantity := q.Put(&value) - if !ok { - t.Error("TestStack Get.Fail") - return - } else { - t.Logf("TestStack Put value:%d[%v], quantity:%v\n", &value, value, quantity) - } - - val, ok, quantity := q.Get() - if !ok { - t.Error("TestStack Get.Fail") - return - } else { - t.Logf("TestStack Get value:%d[%v], quantity:%v\n", val, *(val.(*int)), quantity) - } - if q := q.Quantity(); q != 0 { - t.Errorf("Quantity Error: [%v] <>[%v]", q, 0) - } -} - -func TestQueuePutGet(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - runtime.GOMAXPROCS(runtime.NumCPU()) - const ( - isPrintf = false - ) - - cnt := 10000 - sum := 0 - start := time.Now() - var putD, getD time.Duration - for i := 0; i <= runtime.NumCPU()*4; i++ { - sum += i * cnt - put, get := testQueuePutGet(t, i, cnt) - putD += put - getD += get - } - end := time.Now() - use := end.Sub(start) - op := use / time.Duration(sum) - t.Logf("Grp: %d, Times: %d, use: %v, %v/op", runtime.NumCPU()*4, sum, use, op) - t.Logf("Put: %d, use: %v, %v/op", sum, putD, putD/time.Duration(sum)) - t.Logf("Get: %d, use: %v, %v/op", sum, getD, getD/time.Duration(sum)) -} - -func TestQueueGeneral(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - runtime.GOMAXPROCS(runtime.NumCPU()) - const ( - isPrintf = false - ) - - var miss, Sum int - var Use time.Duration - for i := 1; i <= runtime.NumCPU()*4; i++ { - cnt := 10000 * 100 - if i > 9 { - cnt = 10000 * 10 - } - sum := i * cnt - start := time.Now() - miss = testQueueGeneral(t, i, cnt) - end := time.Now() - use := end.Sub(start) - op := use / time.Duration(sum) - fmt.Printf("%v, Grp: %3d, Times: %10d, miss:%6v, use: %12v, %8v/op\n", - runtime.Version(), i, sum, miss, use, op) - Use += use - Sum += sum - } - op := Use / time.Duration(Sum) - fmt.Printf("%v, Grp: %3v, Times: %10d, miss:%6v, use: %12v, %8v/op\n", - runtime.Version(), "Sum", Sum, 0, Use, op) -} - -func TestQueuePutGoGet(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - var Sum, miss int - var Use time.Duration - for i := 1; i <= runtime.NumCPU()*4; i++ { - // for i := 2; i <= 2; i++ { - cnt := 10000 * 100 - if i > 9 { - cnt = 10000 * 10 - } - sum := i * cnt - start := time.Now() - miss = testQueuePutGoGet(t, i, cnt) - - end := time.Now() - use := end.Sub(start) - op := use / time.Duration(sum) - fmt.Printf("%v, Grp: %3d, Times: %10d, miss:%6v, use: %12v, %8v/op\n", - runtime.Version(), i, sum, miss, use, op) - Use += use - Sum += sum - } - op := Use / time.Duration(Sum) - fmt.Printf("%v, Grp: %3v, Times: %10d, miss:%6v, use: %12v, %8v/op\n", - runtime.Version(), "Sum", Sum, 0, Use, op) -} - -func TestQueuePutDoGet(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - runtime.GOMAXPROCS(runtime.NumCPU()) - - var miss, Sum int - var Use time.Duration - for i := 1; i <= runtime.NumCPU()*4; i++ { - // for i := 2; i <= 2; i++ { - cnt := 10000 * 100 - if i > 9 { - cnt = 10000 * 10 - } - sum := i * cnt - start := time.Now() - miss = testQueuePutDoGet(t, i, cnt) - end := time.Now() - use := end.Sub(start) - op := use / time.Duration(sum) - fmt.Printf("%v, Grp: %3d, Times: %10d, miss:%6v, use: %12v, %8v/op\n", - runtime.Version(), i, sum, miss, use, op) - Use += use - Sum += sum - } - op := Use / time.Duration(Sum) - fmt.Printf("%v, Grp: %3v, Times: %10d, miss:%6v, use: %12v, %8v/op\n", - runtime.Version(), "Sum", Sum, 0, Use, op) -} - -func testQueuePutGet(t *testing.T, grp, cnt int) ( - put time.Duration, get time.Duration) { - var wg sync.WaitGroup - var id int32 - wg.Add(grp) - q := NewQueue(1024 * 1024) - start := time.Now() - for i := 0; i < grp; i++ { - go func(g int) { - defer wg.Done() - for j := 0; j < cnt; j++ { - val := fmt.Sprintf("Node.%d.%d.%d", g, j, atomic.AddInt32(&id, 1)) - ok, _ := q.Put(&val) - for !ok { - time.Sleep(time.Microsecond) - ok, _ = q.Put(&val) - } - } - }(i) - } - wg.Wait() - end := time.Now() - put = end.Sub(start) - - wg.Add(grp) - start = time.Now() - for i := 0; i < grp; i++ { - go func() { - defer wg.Done() - for j := 0; j < cnt; { - _, ok, _ := q.Get() - if !ok { - runtime.Gosched() - } else { - j++ - } - } - }() - } - wg.Wait() - end = time.Now() - get = end.Sub(start) - if q := q.Quantity(); q != 0 { - t.Errorf("Grp:%v, Quantity Error: [%v] <>[%v]", grp, q, 0) - } - return put, get -} - -func testQueueGeneral(t *testing.T, grp, cnt int) int { - - var wg sync.WaitGroup - var idPut, idGet int32 - var miss int32 - - wg.Add(grp) - q := NewQueue(1024 * 1024) - for i := 0; i < grp; i++ { - go func(g int) { - defer wg.Done() - for j := 0; j < cnt; j++ { - val := fmt.Sprintf("Node.%d.%d.%d", g, j, atomic.AddInt32(&idPut, 1)) - ok, _ := q.Put(&val) - for !ok { - time.Sleep(time.Microsecond) - ok, _ = q.Put(&val) - } - } - }(i) - } - - wg.Add(grp) - for i := 0; i < grp; i++ { - go func(g int) { - defer wg.Done() - ok := false - for j := 0; j < cnt; j++ { - _, ok, _ = q.Get() //该语句注释掉将导致运行结果不正确 - for !ok { - atomic.AddInt32(&miss, 1) - time.Sleep(time.Microsecond * 50) - _, ok, _ = q.Get() - } - atomic.AddInt32(&idGet, 1) - } - }(i) - } - wg.Wait() - if q := q.Quantity(); q != 0 { - t.Errorf("Grp:%v, Quantity Error: [%v] <>[%v]", grp, q, 0) - } - return int(miss) -} - -type QtObj struct { - getMiss int32 - putMiss int32 - putCnt int32 - getCnt int32 -} - -type QtSum struct { - Go []QtObj -} - -func newQtSum(grp int) *QtSum { - qt := new(QtSum) - qt.Go = make([]QtObj, grp) - return qt -} - -func (q *QtSum) GetMiss() (num int32) { - for i := range q.Go { - num += q.Go[i].getMiss - } - return -} -func (q *QtSum) PutMiss() (num int32) { - for i := range q.Go { - num += q.Go[i].putMiss - } - return -} -func (q *QtSum) PutCnt() (num int32) { - for i := range q.Go { - num += q.Go[i].putCnt - } - return -} -func (q *QtSum) GetCnt() (num int32) { - for i := range q.Go { - num += q.Go[i].getCnt - } - return -} - -var ( - value int = 1 -) - -func testQueuePutGoGet(t *testing.T, grp, cnt int) int { - var wg sync.WaitGroup - //var Qt = newQtSum(grp) - wg.Add(grp) - q := NewQueue(1024 * 1024) - for i := 0; i < grp; i++ { - go func(g int) { - ok := false - for j := 0; j < cnt; j++ { - ok, _ = q.Put(&value) - //var miss int32 - for !ok { - //Qt.Go[g].getMiss++ - //atomic.AddInt32(&miss, 1) - //time.Sleep(time.Microsecond) - ok, _ = q.Put(&value) - //if miss > 10000 { - // panic(fmt.Sprintf("Put Fail PutId:%12v, GetId:%12v, "+ - // "putCnt:%12v, putMis:%12v, "+ - // "getCnt:%12v, getMis:%12v\n", - // q.eqPut, q.eqGet, Qt.PutCnt(), Qt.PutMiss(), Qt.GetCnt(), Qt.GetMiss())) - //} - } - //Qt.Go[g].putCnt++ - } - wg.Done() - }(i) - } - wg.Add(grp) - for i := 0; i < grp; i++ { - go func(g int) { - ok := false - for j := 0; j < cnt; j++ { - //var miss int32 - _, ok, _ = q.Get() //该语句注释掉将导致运行结果不正确 - for !ok { - //Qt.Go[g].putMiss++ - //atomic.AddInt32(&miss, 1) - //time.Sleep(time.Microsecond * 100) - _, ok, _ = q.Get() - //if miss > 10000 { - // panic(fmt.Sprintf("Get Miss PutId:%12v, GetId:%12v, "+ - // "putCnt:%12v, putMis:%12v, "+ - // "getCnt:%12v, getMis:%12v\n", - // q.eqPut, q.eqGet, Qt.PutCnt(), Qt.PutMiss(), - // Qt.GetCnt(), Qt.GetMiss())) - //} - //printf("Get.Fail\n") - } - //Qt.Go[g].getCnt++ - } - wg.Done() - }(i) - } - wg.Wait() - return 0 //int(Qt.PutMiss()) + int(Qt.GetMiss()) -} - -func testQueuePutDoGet(t *testing.T, grp, cnt int) int { - var wg sync.WaitGroup - //var Qt = newQtSum(grp) - wg.Add(grp) - q := NewQueue(1024 * 1024) - for i := 0; i < grp; i++ { - go func(g int) { - ok := false - for j := 0; j < cnt; j++ { - ok, _ = q.Put(&value) - //var missPut int32 - for !ok { - //Qt.Go[g].getMiss++ - //missPut++ - //time.Sleep(time.Microsecond) - ok, _ = q.Put(&value) - //if missPut > 10000 { - // panic(fmt.Sprintf("Put Fail PutId:%12v, GetId:%12v, "+ - // "putCnt:%12v, putMis:%12v, "+ - // "getCnt:%12v, getMis:%12v\n", - // q.eqPut, q.eqGet, Qt.PutCnt(), Qt.PutMiss(), Qt.GetCnt(), Qt.GetMiss())) - //} - } - //Qt.Go[g].putCnt++ - - //var missGet int32 - _, ok, _ = q.Get() //该语句注释掉将导致运行结果不正确 - for !ok { - //Qt.Go[g].putMiss++ - //missGet++ - //time.Sleep(time.Microsecond * 100) - _, ok, _ = q.Get() - //if missGet > 10000 { - // panic(fmt.Sprintf("Get Miss PutId:%12v, GetId:%12v, "+ - // "putCnt:%12v, putMis:%12v, "+ - // "getCnt:%12v, getMis:%12v\n", - // q.eqPut, q.eqGet, Qt.PutCnt(), Qt.PutMiss(), - // Qt.GetCnt(), Qt.GetMiss())) - //} - //printf("Get.Fail\n") - } - //Qt.Go[g].getCnt++ - } - wg.Done() - }(i) - } - wg.Wait() - return 0 //int(Qt.PutMiss()) + int(Qt.GetMiss()) -} - -func testQueuePutGetOrder(t *testing.T, grp, cnt int) ( - residue int) { - var wg sync.WaitGroup - var idPut, idGet int32 - wg.Add(grp) - q := NewQueue(1024 * 1024) - for i := 0; i < grp; i++ { - go func(g int) { - defer wg.Done() - for j := 0; j < cnt; j++ { - v := atomic.AddInt32(&idPut, 1) - ok, _ := q.Put(v) - for !ok { - time.Sleep(time.Microsecond) - ok, _ = q.Put(v) - } - } - }(i) - } - wg.Wait() - wg.Add(grp) - for i := 0; i < grp; i++ { - go func() { - defer wg.Done() - for j := 0; j < cnt; { - val, ok, _ := q.Get() - if !ok { - fmt.Printf("Get.Fail\n") - runtime.Gosched() - } else { - j++ - idGet++ - if idGet != val.(int32) { - t.Logf("Get.Err %d <> %d\n", idGet, val) - } - } - } - }() - } - wg.Wait() - return -} - -func TestQueuePutGetOrder(t *testing.T) { - runtime.GOMAXPROCS(runtime.NumCPU()) - grp := 1 - cnt := 100 - - testQueuePutGetOrder(t, grp, cnt) - t.Logf("Grp: %d, Times: %d", grp, cnt) -} diff --git a/lib/router/router_test.go b/lib/router/router_test.go index 31a3c87cb..efa0aa947 100644 --- a/lib/router/router_test.go +++ b/lib/router/router_test.go @@ -3,7 +3,6 @@ package router import ( "bufio" "bytes" - "fmt" "io/ioutil" "math/rand" "net" @@ -12,7 +11,6 @@ import ( "testing" "time" - gbytes "github.com/savsgio/gotils/bytes" "infini.sh/framework/lib/fasthttp" ) @@ -46,10 +44,6 @@ func randomHTTPMethod() string { return method } -func buildLocation(host, path string) string { - return fmt.Sprintf("http://%s%s", host, path) -} - var zeroTCPAddr = &net.TCPAddr{ IP: net.IPv4zero, } @@ -439,311 +433,6 @@ func TestRouterMutable(t *testing.T) { } -func TestRouterOPTIONS(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - handlerFunc := func(_ *fasthttp.RequestCtx) {} - - router := New() - router.POST("/path", handlerFunc) - - ctx := new(fasthttp.RequestCtx) - - var checkHandling = func(path, expectedAllowed string, expectedStatusCode int) { - ctx.Request.Header.SetMethod(fasthttp.MethodOptions) - ctx.Request.SetRequestURI(path) - router.Handler(ctx) - - if !(ctx.Response.StatusCode() == expectedStatusCode) { - t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", ctx.Response.StatusCode(), ctx.Response.Header.String()) - } else if allow := string(ctx.Response.Header.Peek("Allow")); allow != expectedAllowed { - t.Error("unexpected Allow header value: " + allow) - } - } - - // test not allowed - // * (server) - checkHandling("*", "OPTIONS, POST", fasthttp.StatusOK) - - // path - checkHandling("/path", "OPTIONS, POST", fasthttp.StatusOK) - - ctx.Request.Header.SetMethod(fasthttp.MethodOptions) - ctx.Request.SetRequestURI("/doesnotexist") - router.Handler(ctx) - if !(ctx.Response.StatusCode() == fasthttp.StatusNotFound) { - t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", ctx.Response.StatusCode(), ctx.Response.Header.String()) - } - - // add another method - router.GET("/path", handlerFunc) - - // set a global OPTIONS handler - router.GlobalOPTIONS = func(ctx *fasthttp.RequestCtx) { - // Adjust status code to 204 - ctx.SetStatusCode(fasthttp.StatusNoContent) - } - - // test again - // * (server) - checkHandling("*", "GET, OPTIONS, POST", fasthttp.StatusNoContent) - - // path - checkHandling("/path", "GET, OPTIONS, POST", fasthttp.StatusNoContent) - - // custom handler - var custom bool - router.OPTIONS("/path", func(ctx *fasthttp.RequestCtx) { - custom = true - }) - - // test again - // * (server) - checkHandling("*", "GET, OPTIONS, POST", fasthttp.StatusNoContent) - if custom { - t.Error("custom handler called on *") - } - - // path - ctx.Request.Header.SetMethod(fasthttp.MethodOptions) - ctx.Request.SetRequestURI("/path") - router.Handler(ctx) - if !(ctx.Response.StatusCode() == fasthttp.StatusNoContent) { - t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", ctx.Response.StatusCode(), ctx.Response.Header.String()) - } - if !custom { - t.Error("custom handler not called") - } -} - -func TestRouterNotAllowed(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - handlerFunc := func(_ *fasthttp.RequestCtx) {} - - router := New() - router.POST("/path", handlerFunc) - - ctx := new(fasthttp.RequestCtx) - - var checkHandling = func(path, expectedAllowed string, expectedStatusCode int) { - ctx.Request.Header.SetMethod(fasthttp.MethodGet) - ctx.Request.SetRequestURI(path) - router.Handler(ctx) - - if !(ctx.Response.StatusCode() == expectedStatusCode) { - t.Errorf("NotAllowed handling failed:: Code=%d, Header=%v", ctx.Response.StatusCode(), ctx.Response.Header.String()) - } else if allow := string(ctx.Response.Header.Peek("Allow")); allow != expectedAllowed { - t.Error("unexpected Allow header value: " + allow) - } - } - - // test not allowed - checkHandling("/path", "OPTIONS, POST", fasthttp.StatusMethodNotAllowed) - - // add another method - router.DELETE("/path", handlerFunc) - router.OPTIONS("/path", handlerFunc) // must be ignored - - // test again - checkHandling("/path", "DELETE, OPTIONS, POST", fasthttp.StatusMethodNotAllowed) - - // test custom handler - responseText := "custom method" - router.MethodNotAllowed = func(ctx *fasthttp.RequestCtx) { - ctx.SetStatusCode(fasthttp.StatusTeapot) - ctx.Write([]byte(responseText)) - } - - ctx.Response.Reset() - router.Handler(ctx) - - if got := string(ctx.Response.Body()); !(got == responseText) { - t.Errorf("unexpected response got %q want %q", got, responseText) - } - if ctx.Response.StatusCode() != fasthttp.StatusTeapot { - t.Errorf("unexpected response code %d want %d", ctx.Response.StatusCode(), fasthttp.StatusTeapot) - } - if allow := string(ctx.Response.Header.Peek("Allow")); allow != "DELETE, OPTIONS, POST" { - t.Error("unexpected Allow header value: " + allow) - } -} - -func testRouterNotFoundByMethod(t *testing.T, method string) { - handlerFunc := func(_ *fasthttp.RequestCtx) {} - host := "fast" - - router := New() - router.Handle(method, "/path", handlerFunc) - router.Handle(method, "/dir/", handlerFunc) - router.Handle(method, "/", handlerFunc) - router.Handle(method, "/{proc}/StaTus", handlerFunc) - router.Handle(method, "/USERS/{name}/enTRies/", handlerFunc) - router.Handle(method, "/static/{filepath:*}", handlerFunc) - - // Moved Permanently, request with GET method - expectedCode := fasthttp.StatusMovedPermanently - if method == fasthttp.MethodConnect { - // CONNECT method does not allow redirects, so Not Found (404) - expectedCode = fasthttp.StatusNotFound - } else if method != fasthttp.MethodGet { - // Permanent Redirect, request with same method - expectedCode = fasthttp.StatusPermanentRedirect - } - - type testRoute struct { - route string - code int - location string - } - - testRoutes := []testRoute{ - {"", fasthttp.StatusOK, ""}, // TSR +/ (Not clean by router, this path is cleaned by fasthttp `ctx.Path()`) - {"/../path", expectedCode, buildLocation(host, "/path")}, // CleanPath (Not clean by router, this path is cleaned by fasthttp `ctx.Path()`) - {"/nope", fasthttp.StatusNotFound, ""}, // NotFound - } - - if method != fasthttp.MethodConnect { - testRoutes = append(testRoutes, []testRoute{ - {"/path/", expectedCode, buildLocation(host, "/path")}, // TSR -/ - {"/dir", expectedCode, buildLocation(host, "/dir/")}, // TSR +/ - {"/PATH", expectedCode, buildLocation(host, "/path")}, // Fixed Case - {"/DIR/", expectedCode, buildLocation(host, "/dir/")}, // Fixed Case - {"/PATH/", expectedCode, buildLocation(host, "/path")}, // Fixed Case -/ - {"/DIR", expectedCode, buildLocation(host, "/dir/")}, // Fixed Case +/ - {"/paTh/?name=foo", expectedCode, buildLocation(host, "/path?name=foo")}, // Fixed Case With Query Params +/ - {"/paTh?name=foo", expectedCode, buildLocation(host, "/path?name=foo")}, // Fixed Case With Query Params +/ - {"/sergio/status/", expectedCode, buildLocation(host, "/sergio/StaTus")}, // Fixed Case With Params -/ - {"/users/atreugo/eNtriEs", expectedCode, buildLocation(host, "/USERS/atreugo/enTRies/")}, // Fixed Case With Params +/ - {"/STatiC/test.go", expectedCode, buildLocation(host, "/static/test.go")}, // Fixed Case Wildcard - }...) - } - - reqMethod := method - if method == MethodWild { - reqMethod = randomHTTPMethod() - } - - for _, tr := range testRoutes { - ctx := new(fasthttp.RequestCtx) - - ctx.Request.Header.SetMethod(reqMethod) - ctx.Request.SetRequestURI(tr.route) - ctx.Request.SetHost(host) - router.Handler(ctx) - - statusCode := ctx.Response.StatusCode() - location := string(ctx.Response.Header.Peek("Location")) - if !(statusCode == tr.code && (statusCode == fasthttp.StatusNotFound || location == tr.location)) { - t.Errorf("NotFound handling route %s failed: ReqMethod=%s, Code=%d, Header=%v", method, tr.route, statusCode, location) - } - } - - ctx := new(fasthttp.RequestCtx) - - // Test custom not found handler - var notFound bool - router.NotFound = func(ctx *fasthttp.RequestCtx) { - ctx.SetStatusCode(fasthttp.StatusNotFound) - notFound = true - } - - ctx.Request.Header.SetMethod(reqMethod) - ctx.Request.SetRequestURI("/nope") - router.Handler(ctx) - if !(ctx.Response.StatusCode() == fasthttp.StatusNotFound && notFound == true) { - t.Errorf("Custom NotFound handler failed: Code=%d, Header=%v", ctx.Response.StatusCode(), ctx.Response.Header.String()) - } - ctx.Response.Reset() -} - -func TestRouterNotFound(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - for _, method := range httpMethods { - testRouterNotFoundByMethod(t, method) - } - - router := New() - handlerFunc := func(_ *fasthttp.RequestCtx) {} - host := "fast" - ctx := new(fasthttp.RequestCtx) - - // Test other method than GET (want 308 instead of 301) - router.PATCH("/path", handlerFunc) - - ctx.Request.Header.SetMethod(fasthttp.MethodPatch) - ctx.Request.SetRequestURI("/path/?key=val") - ctx.Request.SetHost(host) - router.Handler(ctx) - if !(ctx.Response.StatusCode() == fasthttp.StatusPermanentRedirect && string(ctx.Response.Header.Peek("Location")) == buildLocation(host, "/path?key=val")) { - t.Errorf("Custom NotFound handler failed: Code=%d, Header=%v", ctx.Response.StatusCode(), ctx.Response.Header.String()) - } - ctx.Response.Reset() - - // Test special case where no node for the prefix "/" exists - router = New() - router.GET("/a", handlerFunc) - - ctx.Request.Header.SetMethod(fasthttp.MethodPatch) - ctx.Request.SetRequestURI("/") - router.Handler(ctx) - if !(ctx.Response.StatusCode() == fasthttp.StatusNotFound) { - t.Errorf("NotFound handling route / failed: Code=%d", ctx.Response.StatusCode()) - } -} - -func TestRouterNotFound_MethodWild(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - postFound, anyFound := false, false - - router := New() - router.ANY("/{path:*}", func(ctx *fasthttp.RequestCtx) { anyFound = true }) - router.POST("/specific", func(ctx *fasthttp.RequestCtx) { postFound = true }) - - for i := 0; i < 100; i++ { - router.Handle( - randomHTTPMethod(), - fmt.Sprintf("/%s", gbytes.Rand(make([]byte, 5))), - func(ctx *fasthttp.RequestCtx) {}, - ) - } - - ctx := new(fasthttp.RequestCtx) - var request = func(method, path string) { - ctx.Request.Header.SetMethod(method) - ctx.Request.SetRequestURI(path) - router.Handler(ctx) - } - - for _, method := range httpMethods { - request(method, "/specific") - - if method == fasthttp.MethodPost { - if !postFound { - t.Errorf("Method '%s': not found", method) - } - } else { - if !anyFound { - t.Errorf("Method 'ANY' not found with request method %s", method) - } - } - - status := ctx.Response.StatusCode() - if status != fasthttp.StatusOK { - t.Errorf("Response status code == %d, want %d", status, fasthttp.StatusOK) - } - - postFound, anyFound = false, false - ctx.Response.Reset() - } -} - func TestRouterPanicHandler(t *testing.T) { router := New() panicHandled := false diff --git a/lib/seelog/LICENSE.txt b/lib/seelog/LICENSE.txt new file mode 100644 index 000000000..bd5611d95 --- /dev/null +++ b/lib/seelog/LICENSE.txt @@ -0,0 +1,24 @@ +Copyright (c) 2012, Cloud Instruments Co., Ltd. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Cloud Instruments Co., Ltd. nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/seelog/README.markdown b/lib/seelog/README.markdown new file mode 100644 index 000000000..7dd1ab353 --- /dev/null +++ b/lib/seelog/README.markdown @@ -0,0 +1,116 @@ +Seelog +======= + +Seelog is a powerful and easy-to-learn logging framework that provides functionality for flexible dispatching, filtering, and formatting log messages. +It is natively written in the [Go](http://golang.org/) programming language. + +[![Build Status](https://drone.io/github.com/cihub/seelog/status.png)](https://drone.io/github.com/cihub/seelog/latest) + +Features +------------------ + +* Xml configuring to be able to change logger parameters without recompilation +* Changing configurations on the fly without app restart +* Possibility to set different log configurations for different project files and functions +* Adjustable message formatting +* Simultaneous log output to multiple streams +* Choosing logger priority strategy to minimize performance hit +* Different output writers + * Console writer + * File writer + * Buffered writer (Chunk writer) + * Rolling log writer (Logging with rotation) + * SMTP writer + * Others... (See [Wiki](https://github.com/cihub/seelog/wiki)) +* Log message wrappers (JSON, XML, etc.) +* Global variables and functions for easy usage in standalone apps +* Functions for flexible usage in libraries + +Quick-start +----------- + +```go +package main + +import log "github.com/cihub/seelog" + +func main() { + defer log.Flush() + log.Info("Hello from Seelog!") +} +``` + +Installation +------------ + +If you don't have the Go development environment installed, visit the +[Getting Started](http://golang.org/doc/install.html) document and follow the instructions. Once you're ready, execute the following command: + +``` +go get -u github.com/cihub/seelog +``` + +*IMPORTANT*: If you are not using the latest release version of Go, check out this [wiki page](https://github.com/cihub/seelog/wiki/Notes-on-'go-get') + +Documentation +--------------- + +Seelog has github wiki pages, which contain detailed how-tos references: https://github.com/cihub/seelog/wiki + +Examples +--------------- + +Seelog examples can be found here: [seelog-examples](https://github.com/cihub/seelog-examples) + +Issues +--------------- + +Feel free to push issues that could make Seelog better: https://github.com/cihub/seelog/issues + +Changelog +--------------- +* **v2.6** : Config using code and custom formatters + * Configuration using code in addition to xml (All internal receiver/dispatcher/logger types are now exported). + * Custom formatters. Check [wiki](https://github.com/cihub/seelog/wiki/Custom-formatters) + * Bugfixes and internal improvements. +* **v2.5** : Interaction with other systems. Part 2: custom receivers + * Finished custom receivers feature. Check [wiki](https://github.com/cihub/seelog/wiki/custom-receivers) + * Added 'LoggerFromCustomReceiver' + * Added 'LoggerFromWriterWithMinLevelAndFormat' + * Added 'LoggerFromCustomReceiver' + * Added 'LoggerFromParamConfigAs...' +* **v2.4** : Interaction with other systems. Part 1: wrapping seelog + * Added configurable caller stack skip logic + * Added 'SetAdditionalStackDepth' to 'LoggerInterface' +* **v2.3** : Rethinking 'rolling' receiver + * Reimplemented 'rolling' receiver + * Added 'Max rolls' feature for 'rolling' receiver with type='date' + * Fixed 'rolling' receiver issue: renaming on Windows +* **v2.2** : go1.0 compatibility point [go1.0 tag] + * Fixed internal bugs + * Added 'ANSI n [;k]' format identifier: %EscN + * Made current release go1 compatible +* **v2.1** : Some new features + * Rolling receiver archiving option. + * Added format identifier: %Line + * Smtp: added paths to PEM files directories + * Added format identifier: %FuncShort + * Warn, Error and Critical methods now return an error +* **v2.0** : Second major release. BREAKING CHANGES. + * Support of binaries with stripped symbols + * Added log strategy: adaptive + * Critical message now forces Flush() + * Added predefined formats: xml-debug, xml-debug-short, xml, xml-short, json-debug, json-debug-short, json, json-short, debug, debug-short, fast + * Added receiver: conn (network connection writer) + * BREAKING CHANGE: added Tracef, Debugf, Infof, etc. to satisfy the print/printf principle + * Bug fixes +* **v1.0** : Initial release. Features: + * Xml config + * Changing configurations on the fly without app restart + * Contraints and exceptions + * Formatting + * Log strategies: sync, async loop, async timer + * Receivers: buffered, console, file, rolling, smtp + + + diff --git a/lib/seelog/behavior_adaptive_test.go b/lib/seelog/behavior_adaptive_test.go new file mode 100644 index 000000000..e99194930 --- /dev/null +++ b/lib/seelog/behavior_adaptive_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "strconv" + "testing" +) + +func countSequencedRowsInFile(filePath string) (int64, error) { + bts, err := ioutil.ReadFile(filePath) + if err != nil { + return 0, err + } + + bufReader := bufio.NewReader(bytes.NewBuffer(bts)) + + var gotCounter int64 + for { + line, _, bufErr := bufReader.ReadLine() + if bufErr != nil && bufErr != io.EOF { + return 0, bufErr + } + + lineString := string(line) + if lineString == "" { + break + } + + intVal, atoiErr := strconv.ParseInt(lineString, 10, 64) + if atoiErr != nil { + return 0, atoiErr + } + + if intVal != gotCounter { + return 0, fmt.Errorf("wrong order: %d Expected: %d\n", intVal, gotCounter) + } + + gotCounter++ + } + + return gotCounter, nil +} + +func Test_Adaptive(t *testing.T) { + fileName := "beh_test_adaptive.log" + count := 100 + + Current.Close() + + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + return + } + defer func() { + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + } + }() + + testConfig := ` + + + + + + + +` + + logger, _ := LoggerFromConfigAsString(testConfig) + + err := ReplaceLogger(logger) + if err != nil { + t.Error(err) + return + } + + for i := 0; i < count; i++ { + Trace(strconv.Itoa(i)) + } + + Flush() + + gotCount, err := countSequencedRowsInFile(fileName) + if err != nil { + t.Error(err) + return + } + + if int64(count) != gotCount { + t.Errorf("wrong count of log messages. Expected: %v, got: %v.", count, gotCount) + return + } + + Current.Close() +} diff --git a/lib/seelog/behavior_adaptivelogger.go b/lib/seelog/behavior_adaptivelogger.go new file mode 100644 index 000000000..09bb53aea --- /dev/null +++ b/lib/seelog/behavior_adaptivelogger.go @@ -0,0 +1,130 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "fmt" + "math" + "time" +) + +var ( + adaptiveLoggerMaxInterval = time.Minute + adaptiveLoggerMaxCriticalMsgCount = uint32(1000) +) + +// asyncAdaptiveLogger represents asynchronous adaptive logger which acts like +// an async timer logger, but its interval depends on the current message count +// in the queue. +// +// Interval = I, minInterval = m, maxInterval = M, criticalMsgCount = C, msgCount = c: +// I = m + (C - Min(c, C)) / C * (M - m) +type asyncAdaptiveLogger struct { + asyncLogger + minInterval time.Duration + criticalMsgCount uint32 + maxInterval time.Duration +} + +// NewAsyncLoopLogger creates a new asynchronous adaptive logger +func NewAsyncAdaptiveLogger( + config *logConfig, + minInterval time.Duration, + maxInterval time.Duration, + criticalMsgCount uint32) (*asyncAdaptiveLogger, error) { + + if minInterval <= 0 { + return nil, errors.New("async adaptive logger min interval should be > 0") + } + + if maxInterval > adaptiveLoggerMaxInterval { + return nil, fmt.Errorf("async adaptive logger max interval should be <= %s", + adaptiveLoggerMaxInterval) + } + + if criticalMsgCount <= 0 { + return nil, errors.New("async adaptive logger critical msg count should be > 0") + } + + if criticalMsgCount > adaptiveLoggerMaxCriticalMsgCount { + return nil, fmt.Errorf("async adaptive logger critical msg count should be <= %s", + adaptiveLoggerMaxInterval) + } + + asnAdaptiveLogger := new(asyncAdaptiveLogger) + + asnAdaptiveLogger.asyncLogger = *newAsyncLogger(config) + asnAdaptiveLogger.minInterval = minInterval + asnAdaptiveLogger.maxInterval = maxInterval + asnAdaptiveLogger.criticalMsgCount = criticalMsgCount + + go asnAdaptiveLogger.processQueue() + + return asnAdaptiveLogger, nil +} + +func (asnAdaptiveLogger *asyncAdaptiveLogger) processItem() (closed bool, itemCount int) { + asnAdaptiveLogger.queueHasElements.L.Lock() + defer asnAdaptiveLogger.queueHasElements.L.Unlock() + + for asnAdaptiveLogger.msgQueue.Len() == 0 && !asnAdaptiveLogger.Closed() { + asnAdaptiveLogger.queueHasElements.Wait() + } + + if asnAdaptiveLogger.Closed() { + return true, asnAdaptiveLogger.msgQueue.Len() + } + + asnAdaptiveLogger.processQueueElement() + return false, asnAdaptiveLogger.msgQueue.Len() - 1 +} + +// I = m + (C - Min(c, C)) / C * (M - m) => +// I = m + cDiff * mDiff, +// +// cDiff = (C - Min(c, C)) / C) +// mDiff = (M - m) +func (asnAdaptiveLogger *asyncAdaptiveLogger) calcAdaptiveInterval(msgCount int) time.Duration { + critCountF := float64(asnAdaptiveLogger.criticalMsgCount) + cDiff := (critCountF - math.Min(float64(msgCount), critCountF)) / critCountF + mDiff := float64(asnAdaptiveLogger.maxInterval - asnAdaptiveLogger.minInterval) + + return asnAdaptiveLogger.minInterval + time.Duration(cDiff*mDiff) +} + +func (asnAdaptiveLogger *asyncAdaptiveLogger) processQueue() { + for !asnAdaptiveLogger.Closed() { + closed, itemCount := asnAdaptiveLogger.processItem() + + if closed { + break + } + + interval := asnAdaptiveLogger.calcAdaptiveInterval(itemCount) + + <-time.After(interval) + } +} diff --git a/lib/seelog/behavior_asynclogger.go b/lib/seelog/behavior_asynclogger.go new file mode 100644 index 000000000..75231067b --- /dev/null +++ b/lib/seelog/behavior_asynclogger.go @@ -0,0 +1,142 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "container/list" + "fmt" + "sync" +) + +// MaxQueueSize is the critical number of messages in the queue that result in an immediate flush. +const ( + MaxQueueSize = 10000 +) + +type msgQueueItem struct { + level LogLevel + context LogContextInterface + message fmt.Stringer +} + +// asyncLogger represents common data for all asynchronous loggers +type asyncLogger struct { + commonLogger + msgQueue *list.List + queueHasElements *sync.Cond +} + +// newAsyncLogger creates a new asynchronous logger +func newAsyncLogger(config *logConfig) *asyncLogger { + asnLogger := new(asyncLogger) + + asnLogger.msgQueue = list.New() + asnLogger.queueHasElements = sync.NewCond(new(sync.Mutex)) + + asnLogger.commonLogger = *newCommonLogger(config, asnLogger) + + return asnLogger +} + +func (asnLogger *asyncLogger) innerLog( + level LogLevel, + context LogContextInterface, + message fmt.Stringer) { + + asnLogger.addMsgToQueue(level, context, message) +} + +func (asnLogger *asyncLogger) Close() { + asnLogger.m.Lock() + defer asnLogger.m.Unlock() + + if !asnLogger.Closed() { + asnLogger.flushQueue(true) + asnLogger.config.RootDispatcher.Flush() + + if err := asnLogger.config.RootDispatcher.Close(); err != nil { + reportInternalError(err) + } + + asnLogger.closedM.Lock() + asnLogger.closed = true + asnLogger.closedM.Unlock() + asnLogger.queueHasElements.Broadcast() + } +} + +func (asnLogger *asyncLogger) Flush() { + asnLogger.m.Lock() + defer asnLogger.m.Unlock() + + if !asnLogger.Closed() { + asnLogger.flushQueue(true) + asnLogger.config.RootDispatcher.Flush() + } +} + +func (asnLogger *asyncLogger) flushQueue(lockNeeded bool) { + if lockNeeded { + asnLogger.queueHasElements.L.Lock() + defer asnLogger.queueHasElements.L.Unlock() + } + + for asnLogger.msgQueue.Len() > 0 { + asnLogger.processQueueElement() + } +} + +func (asnLogger *asyncLogger) processQueueElement() { + if asnLogger.msgQueue.Len() > 0 { + backElement := asnLogger.msgQueue.Front() + msg, _ := backElement.Value.(msgQueueItem) + asnLogger.processLogMsg(msg.level, msg.message, msg.context) + asnLogger.msgQueue.Remove(backElement) + } +} + +func (asnLogger *asyncLogger) addMsgToQueue( + level LogLevel, + context LogContextInterface, + message fmt.Stringer) { + + if !asnLogger.Closed() { + asnLogger.queueHasElements.L.Lock() + defer asnLogger.queueHasElements.L.Unlock() + + if asnLogger.msgQueue.Len() >= MaxQueueSize { + fmt.Printf("Seelog queue overflow: more than %v messages in the queue. Flushing.\n", MaxQueueSize) + asnLogger.flushQueue(false) + } + + queueItem := msgQueueItem{level, context, message} + + asnLogger.msgQueue.PushBack(queueItem) + asnLogger.queueHasElements.Broadcast() + } else { + err := fmt.Errorf("queue closed! Cannot process element: %d %#v", level, message) + reportInternalError(err) + } +} diff --git a/lib/seelog/behavior_asyncloop_test.go b/lib/seelog/behavior_asyncloop_test.go new file mode 100644 index 000000000..142c4fcff --- /dev/null +++ b/lib/seelog/behavior_asyncloop_test.go @@ -0,0 +1,133 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "strconv" + "testing" +) + +func Test_Asyncloop(t *testing.T) { + fileName := "beh_test_asyncloop.log" + count := 100 + + Current.Close() + + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + return + } + defer func() { + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + } + }() + + testConfig := ` + + + + + + + +` + + logger, _ := LoggerFromConfigAsString(testConfig) + err := ReplaceLogger(logger) + if err != nil { + t.Error(err) + return + } + + for i := 0; i < count; i++ { + Trace(strconv.Itoa(i)) + } + + Flush() + + gotCount, err := countSequencedRowsInFile(fileName) + if err != nil { + t.Error(err) + return + } + + if int64(count) != gotCount { + t.Errorf("wrong count of log messages. Expected: %v, got: %v.", count, gotCount) + return + } + + Current.Close() +} + +func Test_AsyncloopOff(t *testing.T) { + fileName := "beh_test_asyncloopoff.log" + count := 100 + + Current.Close() + + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + return + } + + testConfig := ` + + + + + + + +` + + logger, _ := LoggerFromConfigAsString(testConfig) + err := ReplaceLogger(logger) + if err != nil { + t.Error(err) + return + } + + for i := 0; i < count; i++ { + Trace(strconv.Itoa(i)) + } + + Flush() + + ex, err := fileExists(fileName) + if err != nil { + t.Error(err) + } + if ex { + t.Errorf("logger at level OFF is not expected to create log file at all.") + defer func() { + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + } + }() + } + + Current.Close() +} diff --git a/lib/seelog/behavior_asynclooplogger.go b/lib/seelog/behavior_asynclooplogger.go new file mode 100644 index 000000000..972467b3f --- /dev/null +++ b/lib/seelog/behavior_asynclooplogger.go @@ -0,0 +1,69 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +// asyncLoopLogger represents asynchronous logger which processes the log queue in +// a 'for' loop +type asyncLoopLogger struct { + asyncLogger +} + +// NewAsyncLoopLogger creates a new asynchronous loop logger +func NewAsyncLoopLogger(config *logConfig) *asyncLoopLogger { + + asnLoopLogger := new(asyncLoopLogger) + + asnLoopLogger.asyncLogger = *newAsyncLogger(config) + + go asnLoopLogger.processQueue() + + return asnLoopLogger +} + +func (asnLoopLogger *asyncLoopLogger) processItem() (closed bool) { + asnLoopLogger.queueHasElements.L.Lock() + defer asnLoopLogger.queueHasElements.L.Unlock() + + for asnLoopLogger.msgQueue.Len() == 0 && !asnLoopLogger.Closed() { + asnLoopLogger.queueHasElements.Wait() + } + + if asnLoopLogger.Closed() { + return true + } + + asnLoopLogger.processQueueElement() + return false +} + +func (asnLoopLogger *asyncLoopLogger) processQueue() { + for !asnLoopLogger.Closed() { + closed := asnLoopLogger.processItem() + + if closed { + break + } + } +} diff --git a/lib/seelog/behavior_asynctimer_test.go b/lib/seelog/behavior_asynctimer_test.go new file mode 100644 index 000000000..37bfa6a90 --- /dev/null +++ b/lib/seelog/behavior_asynctimer_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "strconv" + "testing" +) + +func Test_Asynctimer(t *testing.T) { + fileName := "beh_test_asynctimer.log" + count := 100 + + Current.Close() + + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + return + } + defer func() { + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + } + }() + + testConfig := ` + + + + + + + +` + + logger, _ := LoggerFromConfigAsString(testConfig) + err := ReplaceLogger(logger) + if err != nil { + t.Error(err) + return + } + + for i := 0; i < count; i++ { + Trace(strconv.Itoa(i)) + } + + Flush() + + gotCount, err := countSequencedRowsInFile(fileName) + if err != nil { + t.Error(err) + return + } + + if int64(count) != gotCount { + t.Errorf("wrong count of log messages. Expected: %v, got: %v.", count, gotCount) + return + } + + Current.Close() +} diff --git a/lib/seelog/behavior_asynctimerlogger.go b/lib/seelog/behavior_asynctimerlogger.go new file mode 100644 index 000000000..8118f2050 --- /dev/null +++ b/lib/seelog/behavior_asynctimerlogger.go @@ -0,0 +1,82 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "time" +) + +// asyncTimerLogger represents asynchronous logger which processes the log queue each +// 'duration' nanoseconds +type asyncTimerLogger struct { + asyncLogger + interval time.Duration +} + +// NewAsyncLoopLogger creates a new asynchronous loop logger +func NewAsyncTimerLogger(config *logConfig, interval time.Duration) (*asyncTimerLogger, error) { + + if interval <= 0 { + return nil, errors.New("async logger interval should be > 0") + } + + asnTimerLogger := new(asyncTimerLogger) + + asnTimerLogger.asyncLogger = *newAsyncLogger(config) + asnTimerLogger.interval = interval + + go asnTimerLogger.processQueue() + + return asnTimerLogger, nil +} + +func (asnTimerLogger *asyncTimerLogger) processItem() (closed bool) { + asnTimerLogger.queueHasElements.L.Lock() + defer asnTimerLogger.queueHasElements.L.Unlock() + + for asnTimerLogger.msgQueue.Len() == 0 && !asnTimerLogger.Closed() { + asnTimerLogger.queueHasElements.Wait() + } + + if asnTimerLogger.Closed() { + return true + } + + asnTimerLogger.processQueueElement() + return false +} + +func (asnTimerLogger *asyncTimerLogger) processQueue() { + for !asnTimerLogger.Closed() { + closed := asnTimerLogger.processItem() + + if closed { + break + } + + <-time.After(asnTimerLogger.interval) + } +} diff --git a/lib/seelog/behavior_synclogger.go b/lib/seelog/behavior_synclogger.go new file mode 100644 index 000000000..5a022ebc6 --- /dev/null +++ b/lib/seelog/behavior_synclogger.go @@ -0,0 +1,75 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" +) + +// syncLogger performs logging in the same goroutine where 'Trace/Debug/...' +// func was called +type syncLogger struct { + commonLogger +} + +// NewSyncLogger creates a new synchronous logger +func NewSyncLogger(config *logConfig) *syncLogger { + syncLogger := new(syncLogger) + + syncLogger.commonLogger = *newCommonLogger(config, syncLogger) + + return syncLogger +} + +func (syncLogger *syncLogger) innerLog( + level LogLevel, + context LogContextInterface, + message fmt.Stringer) { + + syncLogger.processLogMsg(level, message, context) +} + +func (syncLogger *syncLogger) Close() { + syncLogger.m.Lock() + defer syncLogger.m.Unlock() + + if !syncLogger.Closed() { + if err := syncLogger.config.RootDispatcher.Close(); err != nil { + reportInternalError(err) + } + syncLogger.closedM.Lock() + syncLogger.closed = true + syncLogger.closedM.Unlock() + } +} + +func (syncLogger *syncLogger) Flush() { + syncLogger.m.Lock() + defer syncLogger.m.Unlock() + + if !syncLogger.Closed() { + syncLogger.config.RootDispatcher.Flush() + } +} diff --git a/lib/seelog/behavior_synclogger_test.go b/lib/seelog/behavior_synclogger_test.go new file mode 100644 index 000000000..ddcbbb60b --- /dev/null +++ b/lib/seelog/behavior_synclogger_test.go @@ -0,0 +1,81 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "strconv" + "testing" +) + +func Test_Sync(t *testing.T) { + fileName := "beh_test_sync.log" + count := 100 + + Current.Close() + + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + return + } + defer func() { + if e := tryRemoveFile(fileName); e != nil { + t.Error(e) + } + }() + + testConfig := ` + + + + + + + +` + + logger, _ := LoggerFromConfigAsString(testConfig) + err := ReplaceLogger(logger) + if err != nil { + t.Error(err) + return + } + + for i := 0; i < count; i++ { + Trace(strconv.Itoa(i)) + } + + gotCount, err := countSequencedRowsInFile(fileName) + if err != nil { + t.Error(err) + return + } + + if int64(count) != gotCount { + t.Errorf("wrong count of log messages. Expected: %v, got: %v.", count, gotCount) + return + } + + Current.Close() +} diff --git a/lib/seelog/cfg_config.go b/lib/seelog/cfg_config.go new file mode 100644 index 000000000..c7d848126 --- /dev/null +++ b/lib/seelog/cfg_config.go @@ -0,0 +1,188 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "bytes" + "encoding/xml" + "io" + "os" +) + +// LoggerFromConfigAsFile creates logger with config from file. File should contain valid seelog xml. +func LoggerFromConfigAsFile(fileName string) (LoggerInterface, error) { + file, err := os.Open(fileName) + if err != nil { + return nil, err + } + defer file.Close() + + conf, err := configFromReader(file) + if err != nil { + return nil, err + } + + return createLoggerFromFullConfig(conf) +} + +// LoggerFromConfigAsBytes creates a logger with config from bytes stream. Bytes should contain valid seelog xml. +func LoggerFromConfigAsBytes(data []byte) (LoggerInterface, error) { + conf, err := configFromReader(bytes.NewBuffer(data)) + if err != nil { + return nil, err + } + + return createLoggerFromFullConfig(conf) +} + +// LoggerFromConfigAsString creates a logger with config from a string. String should contain valid seelog xml. +func LoggerFromConfigAsString(data string) (LoggerInterface, error) { + return LoggerFromConfigAsBytes([]byte(data)) +} + +// LoggerFromParamConfigAsFile does the same as LoggerFromConfigAsFile, but includes special parser options. +// See 'CfgParseParams' comments. +func LoggerFromParamConfigAsFile(fileName string, parserParams *CfgParseParams) (LoggerInterface, error) { + file, err := os.Open(fileName) + if err != nil { + return nil, err + } + defer file.Close() + + conf, err := configFromReaderWithConfig(file, parserParams) + if err != nil { + return nil, err + } + + return createLoggerFromFullConfig(conf) +} + +// LoggerFromParamConfigAsBytes does the same as LoggerFromConfigAsBytes, but includes special parser options. +// See 'CfgParseParams' comments. +func LoggerFromParamConfigAsBytes(data []byte, parserParams *CfgParseParams) (LoggerInterface, error) { + conf, err := configFromReaderWithConfig(bytes.NewBuffer(data), parserParams) + if err != nil { + return nil, err + } + + return createLoggerFromFullConfig(conf) +} + +// LoggerFromParamConfigAsString does the same as LoggerFromConfigAsString, but includes special parser options. +// See 'CfgParseParams' comments. +func LoggerFromParamConfigAsString(data string, parserParams *CfgParseParams) (LoggerInterface, error) { + return LoggerFromParamConfigAsBytes([]byte(data), parserParams) +} + +// LoggerFromWriterWithMinLevel is shortcut for LoggerFromWriterWithMinLevelAndFormat(output, minLevel, DefaultMsgFormat) +func LoggerFromWriterWithMinLevel(output io.Writer, minLevel LogLevel) (LoggerInterface, error) { + return LoggerFromWriterWithMinLevelAndFormat(output, minLevel, DefaultMsgFormat) +} + +// LoggerFromWriterWithMinLevelAndFormat creates a proxy logger that uses io.Writer as the +// receiver with minimal level = minLevel and with specified format. +// +// All messages with level more or equal to minLevel will be written to output and +// formatted using the default seelog format. +// +// Can be called for usage with non-Seelog systems +func LoggerFromWriterWithMinLevelAndFormat(output io.Writer, minLevel LogLevel, format string) (LoggerInterface, error) { + constraints, err := NewMinMaxConstraints(minLevel, CriticalLvl) + if err != nil { + return nil, err + } + formatter, err := NewFormatter(format) + if err != nil { + return nil, err + } + dispatcher, err := NewSplitDispatcher(formatter, []interface{}{output}) + if err != nil { + return nil, err + } + + conf, err := newFullLoggerConfig(constraints, make([]*LogLevelException, 0), dispatcher, syncloggerTypeFromString, nil, nil) + if err != nil { + return nil, err + } + + return createLoggerFromFullConfig(conf) +} + +// LoggerFromXMLDecoder creates logger with config from a XML decoder starting from a specific node. +// It should contain valid seelog xml, except for root node name. +func LoggerFromXMLDecoder(xmlParser *xml.Decoder, rootNode xml.Token) (LoggerInterface, error) { + conf, err := configFromXMLDecoder(xmlParser, rootNode) + if err != nil { + return nil, err + } + + return createLoggerFromFullConfig(conf) +} + +// LoggerFromCustomReceiver creates a proxy logger that uses a CustomReceiver as the +// receiver. +// +// All messages will be sent to the specified custom receiver without additional +// formatting ('%Msg' format is used). +// +// Check CustomReceiver, RegisterReceiver for additional info. +// +// NOTE 1: CustomReceiver.AfterParse is only called when a receiver is instantiated +// by the config parser while parsing config. So, if you are not planning to use the +// same CustomReceiver for both proxying (via LoggerFromCustomReceiver call) and +// loading from config, just leave AfterParse implementation empty. +// +// NOTE 2: Unlike RegisterReceiver, LoggerFromCustomReceiver takes an already initialized +// instance that implements CustomReceiver. So, fill it with data and perform any initialization +// logic before calling this func and it won't be lost. +// +// So: +// * RegisterReceiver takes value just to get the reflect.Type from it and then +// instantiate it as many times as config is reloaded. +// +// * LoggerFromCustomReceiver takes value and uses it without modification and +// reinstantiation, directy passing it to the dispatcher tree. +func LoggerFromCustomReceiver(receiver CustomReceiver) (LoggerInterface, error) { + constraints, err := NewMinMaxConstraints(TraceLvl, CriticalLvl) + if err != nil { + return nil, err + } + + output, err := NewCustomReceiverDispatcherByValue(msgonlyformatter, receiver, "user-proxy", CustomReceiverInitArgs{}) + if err != nil { + return nil, err + } + dispatcher, err := NewSplitDispatcher(msgonlyformatter, []interface{}{output}) + if err != nil { + return nil, err + } + + conf, err := newFullLoggerConfig(constraints, make([]*LogLevelException, 0), dispatcher, syncloggerTypeFromString, nil, nil) + if err != nil { + return nil, err + } + + return createLoggerFromFullConfig(conf) +} diff --git a/lib/seelog/cfg_errors.go b/lib/seelog/cfg_errors.go new file mode 100644 index 000000000..c1fb4d101 --- /dev/null +++ b/lib/seelog/cfg_errors.go @@ -0,0 +1,61 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" +) + +var ( + errNodeMustHaveChildren = errors.New("node must have children") + errNodeCannotHaveChildren = errors.New("node cannot have children") +) + +type unexpectedChildElementError struct { + baseError +} + +func newUnexpectedChildElementError(msg string) *unexpectedChildElementError { + custmsg := "Unexpected child element: " + msg + return &unexpectedChildElementError{baseError{message: custmsg}} +} + +type missingArgumentError struct { + baseError +} + +func newMissingArgumentError(nodeName, attrName string) *missingArgumentError { + custmsg := "Output '" + nodeName + "' has no '" + attrName + "' attribute" + return &missingArgumentError{baseError{message: custmsg}} +} + +type unexpectedAttributeError struct { + baseError +} + +func newUnexpectedAttributeError(nodeName, attr string) *unexpectedAttributeError { + custmsg := nodeName + " has unexpected attribute: " + attr + return &unexpectedAttributeError{baseError{message: custmsg}} +} diff --git a/lib/seelog/cfg_logconfig.go b/lib/seelog/cfg_logconfig.go new file mode 100644 index 000000000..6ba6f9a94 --- /dev/null +++ b/lib/seelog/cfg_logconfig.go @@ -0,0 +1,141 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" +) + +type loggerTypeFromString uint8 + +const ( + syncloggerTypeFromString = iota + asyncLooploggerTypeFromString + asyncTimerloggerTypeFromString + adaptiveLoggerTypeFromString + defaultloggerTypeFromString = asyncLooploggerTypeFromString +) + +const ( + syncloggerTypeFromStringStr = "sync" + asyncloggerTypeFromStringStr = "asyncloop" + asyncTimerloggerTypeFromStringStr = "asynctimer" + adaptiveLoggerTypeFromStringStr = "adaptive" +) + +// asyncTimerLoggerData represents specific data for async timer logger +type asyncTimerLoggerData struct { + AsyncInterval uint32 +} + +// adaptiveLoggerData represents specific data for adaptive timer logger +type adaptiveLoggerData struct { + MinInterval uint32 + MaxInterval uint32 + CriticalMsgCount uint32 +} + +var loggerTypeToStringRepresentations = map[loggerTypeFromString]string{ + syncloggerTypeFromString: syncloggerTypeFromStringStr, + asyncLooploggerTypeFromString: asyncloggerTypeFromStringStr, + asyncTimerloggerTypeFromString: asyncTimerloggerTypeFromStringStr, + adaptiveLoggerTypeFromString: adaptiveLoggerTypeFromStringStr, +} + +// getLoggerTypeFromString parses a string and returns a corresponding logger type, if successful. +func getLoggerTypeFromString(logTypeString string) (level loggerTypeFromString, found bool) { + for logType, logTypeStr := range loggerTypeToStringRepresentations { + if logTypeStr == logTypeString { + return logType, true + } + } + + return 0, false +} + +// logConfig stores logging configuration. Contains messages dispatcher, allowed log level rules +// (general constraints and exceptions) +type logConfig struct { + Constraints logLevelConstraints // General log level rules (>min and + + + + + +` + + conf, err := configFromReader(strings.NewReader(testConfig)) + if err != nil { + t.Errorf("parse error: %s\n", err.Error()) + return + } + + context, err := currentContext(nil) + if err != nil { + t.Errorf("cannot get current context:" + err.Error()) + return + } + firstContext, err := getFirstContext() + if err != nil { + t.Errorf("cannot get current context:" + err.Error()) + return + } + secondContext, err := getSecondContext() + if err != nil { + t.Errorf("cannot get current context:" + err.Error()) + return + } + + if !conf.IsAllowed(TraceLvl, context) { + t.Errorf("error: deny trace in current context") + } + if conf.IsAllowed(TraceLvl, firstContext) { + t.Errorf("error: allow trace in first context") + } + if conf.IsAllowed(ErrorLvl, context) { + t.Errorf("error: allow error in current context") + } + if !conf.IsAllowed(ErrorLvl, secondContext) { + t.Errorf("error: deny error in second context") + } + + // cache test + if !conf.IsAllowed(TraceLvl, context) { + t.Errorf("error: deny trace in current context") + } + if conf.IsAllowed(TraceLvl, firstContext) { + t.Errorf("error: allow trace in first context") + } + if conf.IsAllowed(ErrorLvl, context) { + t.Errorf("error: allow error in current context") + } + if !conf.IsAllowed(ErrorLvl, secondContext) { + t.Errorf("error: deny error in second context") + } +} + +func getFirstContext() (LogContextInterface, error) { + return currentContext(nil) +} + +func getSecondContext() (LogContextInterface, error) { + return currentContext(nil) +} diff --git a/lib/seelog/cfg_parser.go b/lib/seelog/cfg_parser.go new file mode 100644 index 000000000..7fb9aabf0 --- /dev/null +++ b/lib/seelog/cfg_parser.go @@ -0,0 +1,1238 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "crypto/tls" + "encoding/xml" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +// Names of elements of seelog config. +const ( + seelogConfigID = "seelog" + outputsID = "outputs" + formatsID = "formats" + minLevelID = "minlevel" + maxLevelID = "maxlevel" + levelsID = "levels" + exceptionsID = "exceptions" + exceptionID = "exception" + funcPatternID = "funcpattern" + filePatternID = "filepattern" + formatID = "format" + formatAttrID = "format" + formatKeyAttrID = "id" + outputFormatID = "formatid" + pathID = "path" + fileWriterID = "file" + smtpWriterID = "smtp" + senderaddressID = "senderaddress" + senderNameID = "sendername" + recipientID = "recipient" + mailHeaderID = "header" + mailHeaderNameID = "name" + mailHeaderValueID = "value" + addressID = "address" + hostNameID = "hostname" + hostPortID = "hostport" + userNameID = "username" + userPassID = "password" + cACertDirpathID = "cacertdirpath" + subjectID = "subject" + splitterDispatcherID = "splitter" + consoleWriterID = "console" + customReceiverID = "custom" + customNameAttrID = "name" + customNameDataAttrPrefix = "data-" + filterDispatcherID = "filter" + filterLevelsAttrID = "levels" + rollingfileWriterID = "rollingfile" + rollingFileTypeAttr = "type" + rollingFilePathAttr = "filename" + rollingFileMaxSizeAttr = "maxsize" + rollingFileMaxRollsAttr = "maxrolls" + rollingFileNameModeAttr = "namemode" + rollingFileDataPatternAttr = "datepattern" + rollingFileArchiveAttr = "archivetype" + rollingFileArchivePathAttr = "archivepath" + bufferedWriterID = "buffered" + bufferedSizeAttr = "size" + bufferedFlushPeriodAttr = "flushperiod" + loggerTypeFromStringAttr = "type" + asyncLoggerIntervalAttr = "asyncinterval" + adaptLoggerMinIntervalAttr = "mininterval" + adaptLoggerMaxIntervalAttr = "maxinterval" + adaptLoggerCriticalMsgCountAttr = "critmsgcount" + predefinedPrefix = "std:" + connWriterID = "conn" + connWriterAddrAttr = "addr" + connWriterNetAttr = "net" + connWriterReconnectOnMsgAttr = "reconnectonmsg" + connWriterUseTLSAttr = "tls" + connWriterInsecureSkipVerifyAttr = "insecureskipverify" +) + +// CustomReceiverProducer is the signature of the function CfgParseParams needs to create +// custom receivers. +type CustomReceiverProducer func(CustomReceiverInitArgs) (CustomReceiver, error) + +// CfgParseParams represent specific parse options or flags used by parser. It is used if seelog parser needs +// some special directives or additional info to correctly parse a config. +type CfgParseParams struct { + // CustomReceiverProducers expose the same functionality as RegisterReceiver func + // but only in the scope (context) of the config parse func instead of a global package scope. + // + // It means that if you use custom receivers in your code, you may either register them globally once with + // RegisterReceiver or you may call funcs like LoggerFromParamConfigAsFile (with 'ParamConfig') + // and use CustomReceiverProducers to provide custom producer funcs. + // + // A producer func is called when config parser processes a '' element. It takes the 'name' attribute + // of the element and tries to find a match in two places: + // 1) CfgParseParams.CustomReceiverProducers map + // 2) Global type map, filled by RegisterReceiver + // + // If a match is found in the CustomReceiverProducers map, parser calls the corresponding producer func + // passing the init args to it. The func takes exactly the same args as CustomReceiver.AfterParse. + // The producer func must return a correct receiver or an error. If case of error, seelog will behave + // in the same way as with any other config error. + // + // You may use this param to set custom producers in case you need to pass some context when instantiating + // a custom receiver or if you frequently change custom receivers with different parameters or in any other + // situation where package-level registering (RegisterReceiver) is not an option for you. + CustomReceiverProducers map[string]CustomReceiverProducer +} + +func (cfg *CfgParseParams) String() string { + return fmt.Sprintf("CfgParams: {custom_recs=%d}", len(cfg.CustomReceiverProducers)) +} + +type elementMapEntry struct { + constructor func(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) +} + +var elementMap map[string]elementMapEntry +var predefinedFormats map[string]*formatter + +func init() { + elementMap = map[string]elementMapEntry{ + fileWriterID: {createfileWriter}, + splitterDispatcherID: {createSplitter}, + customReceiverID: {createCustomReceiver}, + filterDispatcherID: {createFilter}, + consoleWriterID: {createConsoleWriter}, + rollingfileWriterID: {createRollingFileWriter}, + bufferedWriterID: {createbufferedWriter}, + smtpWriterID: {createSMTPWriter}, + connWriterID: {createconnWriter}, + } + + err := fillPredefinedFormats() + if err != nil { + panic(fmt.Sprintf("Seelog couldn't start: predefined formats creation failed. Error: %s", err.Error())) + } +} + +func fillPredefinedFormats() error { + predefinedFormatsWithoutPrefix := map[string]string{ + "xml-debug": `%Lev%Msg%RelFile%Func%Line`, + "xml-debug-short": `%Ns%l%Msg

%RelFile

%Func`, + "xml": `%Lev%Msg`, + "xml-short": `%Ns%l%Msg`, + + "json-debug": `{"time":%Ns,"lev":"%Lev","msg":"%Msg","path":"%RelFile","func":"%Func","line":"%Line"}`, + "json-debug-short": `{"t":%Ns,"l":"%Lev","m":"%Msg","p":"%RelFile","f":"%Func"}`, + "json": `{"time":%Ns,"lev":"%Lev","msg":"%Msg"}`, + "json-short": `{"t":%Ns,"l":"%Lev","m":"%Msg"}`, + + "debug": `[%LEVEL] %RelFile:%Func.%Line %Date %Time %Msg%n`, + "debug-short": `[%LEVEL] %Date %Time %Msg%n`, + "fast": `%Ns %l %Msg%n`, + } + + predefinedFormats = make(map[string]*formatter) + + for formatKey, format := range predefinedFormatsWithoutPrefix { + formatter, err := NewFormatter(format) + if err != nil { + return err + } + + predefinedFormats[predefinedPrefix+formatKey] = formatter + } + + return nil +} + +// configFromXMLDecoder parses data from a given XML decoder. +// Returns parsed config which can be used to create logger in case no errors occured. +// Returns error if format is incorrect or anything happened. +func configFromXMLDecoder(xmlParser *xml.Decoder, rootNode xml.Token) (*configForParsing, error) { + return configFromXMLDecoderWithConfig(xmlParser, rootNode, nil) +} + +// configFromXMLDecoderWithConfig parses data from a given XML decoder. +// Returns parsed config which can be used to create logger in case no errors occured. +// Returns error if format is incorrect or anything happened. +func configFromXMLDecoderWithConfig(xmlParser *xml.Decoder, rootNode xml.Token, cfg *CfgParseParams) (*configForParsing, error) { + _, ok := rootNode.(xml.StartElement) + if !ok { + return nil, errors.New("rootNode must be XML startElement") + } + + config, err := unmarshalNode(xmlParser, rootNode) + if err != nil { + return nil, err + } + if config == nil { + return nil, errors.New("xml has no content") + } + + return configFromXMLNodeWithConfig(config, cfg) +} + +// configFromReader parses data from a given reader. +// Returns parsed config which can be used to create logger in case no errors occured. +// Returns error if format is incorrect or anything happened. +func configFromReader(reader io.Reader) (*configForParsing, error) { + return configFromReaderWithConfig(reader, nil) +} + +// configFromReaderWithConfig parses data from a given reader. +// Returns parsed config which can be used to create logger in case no errors occured. +// Returns error if format is incorrect or anything happened. +func configFromReaderWithConfig(reader io.Reader, cfg *CfgParseParams) (*configForParsing, error) { + config, err := unmarshalConfig(reader) + if err != nil { + return nil, err + } + + if config.name != seelogConfigID { + return nil, errors.New("root xml tag must be '" + seelogConfigID + "'") + } + + return configFromXMLNodeWithConfig(config, cfg) +} + +func configFromXMLNodeWithConfig(config *xmlNode, cfg *CfgParseParams) (*configForParsing, error) { + err := checkUnexpectedAttribute( + config, + minLevelID, + maxLevelID, + levelsID, + loggerTypeFromStringAttr, + asyncLoggerIntervalAttr, + adaptLoggerMinIntervalAttr, + adaptLoggerMaxIntervalAttr, + adaptLoggerCriticalMsgCountAttr, + ) + if err != nil { + return nil, err + } + + err = checkExpectedElements(config, optionalElement(outputsID), optionalElement(formatsID), optionalElement(exceptionsID)) + if err != nil { + return nil, err + } + + constraints, err := getConstraints(config) + if err != nil { + return nil, err + } + + exceptions, err := getExceptions(config) + if err != nil { + return nil, err + } + err = checkDistinctExceptions(exceptions) + if err != nil { + return nil, err + } + + formats, err := getFormats(config) + if err != nil { + return nil, err + } + + dispatcher, err := getOutputsTree(config, formats, cfg) + if err != nil { + // If we open several files, but then fail to parse the config, we should close + // those files before reporting that config is invalid. + if dispatcher != nil { + dispatcher.Close() + } + + return nil, err + } + + loggerType, logData, err := getloggerTypeFromStringData(config) + if err != nil { + return nil, err + } + + return newFullLoggerConfig(constraints, exceptions, dispatcher, loggerType, logData, cfg) +} + +func getConstraints(node *xmlNode) (logLevelConstraints, error) { + minLevelStr, isMinLevel := node.attributes[minLevelID] + maxLevelStr, isMaxLevel := node.attributes[maxLevelID] + levelsStr, isLevels := node.attributes[levelsID] + + if isLevels && (isMinLevel && isMaxLevel) { + return nil, errors.New("for level declaration use '" + levelsID + "'' OR '" + minLevelID + + "', '" + maxLevelID + "'") + } + + offString := LogLevel(Off).String() + + if (isLevels && strings.TrimSpace(levelsStr) == offString) || + (isMinLevel && !isMaxLevel && minLevelStr == offString) { + + return NewOffConstraints() + } + + if isLevels { + levels, err := parseLevels(levelsStr) + if err != nil { + return nil, err + } + return NewListConstraints(levels) + } + + var minLevel = LogLevel(TraceLvl) + if isMinLevel { + found := true + minLevel, found = LogLevelFromString(minLevelStr) + if !found { + return nil, errors.New("declared " + minLevelID + " not found: " + minLevelStr) + } + } + + var maxLevel = LogLevel(CriticalLvl) + if isMaxLevel { + found := true + maxLevel, found = LogLevelFromString(maxLevelStr) + if !found { + return nil, errors.New("declared " + maxLevelID + " not found: " + maxLevelStr) + } + } + + return NewMinMaxConstraints(minLevel, maxLevel) +} + +func parseLevels(str string) ([]LogLevel, error) { + levelsStrArr := strings.Split(strings.Replace(str, " ", "", -1), ",") + var levels []LogLevel + for _, levelStr := range levelsStrArr { + level, found := LogLevelFromString(levelStr) + if !found { + return nil, errors.New("declared level not found: " + levelStr) + } + + levels = append(levels, level) + } + + return levels, nil +} + +func getExceptions(config *xmlNode) ([]*LogLevelException, error) { + var exceptions []*LogLevelException + + var exceptionsNode *xmlNode + for _, child := range config.children { + if child.name == exceptionsID { + exceptionsNode = child + break + } + } + + if exceptionsNode == nil { + return exceptions, nil + } + + err := checkUnexpectedAttribute(exceptionsNode) + if err != nil { + return nil, err + } + + err = checkExpectedElements(exceptionsNode, multipleMandatoryElements("exception")) + if err != nil { + return nil, err + } + + for _, exceptionNode := range exceptionsNode.children { + if exceptionNode.name != exceptionID { + return nil, errors.New("incorrect nested element in exceptions section: " + exceptionNode.name) + } + + err := checkUnexpectedAttribute(exceptionNode, minLevelID, maxLevelID, levelsID, funcPatternID, filePatternID) + if err != nil { + return nil, err + } + + constraints, err := getConstraints(exceptionNode) + if err != nil { + return nil, errors.New("incorrect " + exceptionsID + " node: " + err.Error()) + } + + funcPattern, isFuncPattern := exceptionNode.attributes[funcPatternID] + filePattern, isFilePattern := exceptionNode.attributes[filePatternID] + if !isFuncPattern { + funcPattern = "*" + } + if !isFilePattern { + filePattern = "*" + } + + exception, err := NewLogLevelException(funcPattern, filePattern, constraints) + if err != nil { + return nil, errors.New("incorrect exception node: " + err.Error()) + } + + exceptions = append(exceptions, exception) + } + + return exceptions, nil +} + +func checkDistinctExceptions(exceptions []*LogLevelException) error { + for i, exception := range exceptions { + for j, exception1 := range exceptions { + if i == j { + continue + } + + if exception.FuncPattern() == exception1.FuncPattern() && + exception.FilePattern() == exception1.FilePattern() { + + return fmt.Errorf("there are two or more duplicate exceptions. Func: %v, file %v", + exception.FuncPattern(), exception.FilePattern()) + } + } + } + + return nil +} + +func getFormats(config *xmlNode) (map[string]*formatter, error) { + formats := make(map[string]*formatter, 0) + + var formatsNode *xmlNode + for _, child := range config.children { + if child.name == formatsID { + formatsNode = child + break + } + } + + if formatsNode == nil { + return formats, nil + } + + err := checkUnexpectedAttribute(formatsNode) + if err != nil { + return nil, err + } + + err = checkExpectedElements(formatsNode, multipleMandatoryElements("format")) + if err != nil { + return nil, err + } + + for _, formatNode := range formatsNode.children { + if formatNode.name != formatID { + return nil, errors.New("incorrect nested element in " + formatsID + " section: " + formatNode.name) + } + + err := checkUnexpectedAttribute(formatNode, formatKeyAttrID, formatID) + if err != nil { + return nil, err + } + + id, isID := formatNode.attributes[formatKeyAttrID] + formatStr, isFormat := formatNode.attributes[formatAttrID] + if !isID { + return nil, errors.New("format has no '" + formatKeyAttrID + "' attribute") + } + if !isFormat { + return nil, errors.New("format[" + id + "] has no '" + formatAttrID + "' attribute") + } + + formatter, err := NewFormatter(formatStr) + if err != nil { + return nil, err + } + + formats[id] = formatter + } + + return formats, nil +} + +func getloggerTypeFromStringData(config *xmlNode) (logType loggerTypeFromString, logData interface{}, err error) { + logTypeStr, loggerTypeExists := config.attributes[loggerTypeFromStringAttr] + + if !loggerTypeExists { + return defaultloggerTypeFromString, nil, nil + } + + logType, found := getLoggerTypeFromString(logTypeStr) + + if !found { + return 0, nil, fmt.Errorf("unknown logger type: %s", logTypeStr) + } + + if logType == asyncTimerloggerTypeFromString { + intervalStr, intervalExists := config.attributes[asyncLoggerIntervalAttr] + if !intervalExists { + return 0, nil, newMissingArgumentError(config.name, asyncLoggerIntervalAttr) + } + + interval, err := strconv.ParseUint(intervalStr, 10, 32) + if err != nil { + return 0, nil, err + } + + logData = asyncTimerLoggerData{uint32(interval)} + } else if logType == adaptiveLoggerTypeFromString { + + // Min interval + minIntStr, minIntExists := config.attributes[adaptLoggerMinIntervalAttr] + if !minIntExists { + return 0, nil, newMissingArgumentError(config.name, adaptLoggerMinIntervalAttr) + } + minInterval, err := strconv.ParseUint(minIntStr, 10, 32) + if err != nil { + return 0, nil, err + } + + // Max interval + maxIntStr, maxIntExists := config.attributes[adaptLoggerMaxIntervalAttr] + if !maxIntExists { + return 0, nil, newMissingArgumentError(config.name, adaptLoggerMaxIntervalAttr) + } + maxInterval, err := strconv.ParseUint(maxIntStr, 10, 32) + if err != nil { + return 0, nil, err + } + + // Critical msg count + criticalMsgCountStr, criticalMsgCountExists := config.attributes[adaptLoggerCriticalMsgCountAttr] + if !criticalMsgCountExists { + return 0, nil, newMissingArgumentError(config.name, adaptLoggerCriticalMsgCountAttr) + } + criticalMsgCount, err := strconv.ParseUint(criticalMsgCountStr, 10, 32) + if err != nil { + return 0, nil, err + } + + logData = adaptiveLoggerData{uint32(minInterval), uint32(maxInterval), uint32(criticalMsgCount)} + } + + return logType, logData, nil +} + +func getOutputsTree(config *xmlNode, formats map[string]*formatter, cfg *CfgParseParams) (dispatcherInterface, error) { + var outputsNode *xmlNode + for _, child := range config.children { + if child.name == outputsID { + outputsNode = child + break + } + } + + if outputsNode != nil { + err := checkUnexpectedAttribute(outputsNode, outputFormatID) + if err != nil { + return nil, err + } + + formatter, err := getCurrentFormat(outputsNode, DefaultFormatter, formats) + if err != nil { + return nil, err + } + + output, err := createSplitter(outputsNode, formatter, formats, cfg) + if err != nil { + return nil, err + } + + dispatcher, ok := output.(dispatcherInterface) + if ok { + return dispatcher, nil + } + } + + console, err := NewConsoleWriter() + if err != nil { + return nil, err + } + return NewSplitDispatcher(DefaultFormatter, []interface{}{console}) +} + +func getCurrentFormat(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter) (*formatter, error) { + formatID, isFormatID := node.attributes[outputFormatID] + if !isFormatID { + return formatFromParent, nil + } + + format, ok := formats[formatID] + if ok { + return format, nil + } + + // Test for predefined format match + pdFormat, pdOk := predefinedFormats[formatID] + + if !pdOk { + return nil, errors.New("formatid = '" + formatID + "' doesn't exist") + } + + return pdFormat, nil +} + +func createInnerReceivers(node *xmlNode, format *formatter, formats map[string]*formatter, cfg *CfgParseParams) ([]interface{}, error) { + var outputs []interface{} + for _, childNode := range node.children { + entry, ok := elementMap[childNode.name] + if !ok { + return nil, errors.New("unnknown tag '" + childNode.name + "' in outputs section") + } + + output, err := entry.constructor(childNode, format, formats, cfg) + if err != nil { + return nil, err + } + + outputs = append(outputs, output) + } + + return outputs, nil +} + +func createSplitter(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) { + err := checkUnexpectedAttribute(node, outputFormatID) + if err != nil { + return nil, err + } + + if !node.hasChildren() { + return nil, errNodeMustHaveChildren + } + + currentFormat, err := getCurrentFormat(node, formatFromParent, formats) + if err != nil { + return nil, err + } + + receivers, err := createInnerReceivers(node, currentFormat, formats, cfg) + if err != nil { + return nil, err + } + + return NewSplitDispatcher(currentFormat, receivers) +} + +func createCustomReceiver(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) { + dataCustomPrefixes := make(map[string]string) + // Expecting only 'formatid', 'name' and 'data-' attrs + for attr, attrval := range node.attributes { + isExpected := false + if attr == outputFormatID || + attr == customNameAttrID { + isExpected = true + } + if strings.HasPrefix(attr, customNameDataAttrPrefix) { + dataCustomPrefixes[attr[len(customNameDataAttrPrefix):]] = attrval + isExpected = true + } + if !isExpected { + return nil, newUnexpectedAttributeError(node.name, attr) + } + } + + if node.hasChildren() { + return nil, errNodeCannotHaveChildren + } + customName, hasCustomName := node.attributes[customNameAttrID] + if !hasCustomName { + return nil, newMissingArgumentError(node.name, customNameAttrID) + } + currentFormat, err := getCurrentFormat(node, formatFromParent, formats) + if err != nil { + return nil, err + } + args := CustomReceiverInitArgs{ + XmlCustomAttrs: dataCustomPrefixes, + } + + if cfg != nil && cfg.CustomReceiverProducers != nil { + if prod, ok := cfg.CustomReceiverProducers[customName]; ok { + rec, err := prod(args) + if err != nil { + return nil, err + } + creceiver, err := NewCustomReceiverDispatcherByValue(currentFormat, rec, customName, args) + if err != nil { + return nil, err + } + err = rec.AfterParse(args) + if err != nil { + return nil, err + } + return creceiver, nil + } + } + + return NewCustomReceiverDispatcher(currentFormat, customName, args) +} + +func createFilter(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) { + err := checkUnexpectedAttribute(node, outputFormatID, filterLevelsAttrID) + if err != nil { + return nil, err + } + + if !node.hasChildren() { + return nil, errNodeMustHaveChildren + } + + currentFormat, err := getCurrentFormat(node, formatFromParent, formats) + if err != nil { + return nil, err + } + + levelsStr, isLevels := node.attributes[filterLevelsAttrID] + if !isLevels { + return nil, newMissingArgumentError(node.name, filterLevelsAttrID) + } + + levels, err := parseLevels(levelsStr) + if err != nil { + return nil, err + } + + receivers, err := createInnerReceivers(node, currentFormat, formats, cfg) + if err != nil { + return nil, err + } + + return NewFilterDispatcher(currentFormat, receivers, levels...) +} + +func createfileWriter(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) { + err := checkUnexpectedAttribute(node, outputFormatID, pathID) + if err != nil { + return nil, err + } + + if node.hasChildren() { + return nil, errNodeCannotHaveChildren + } + + currentFormat, err := getCurrentFormat(node, formatFromParent, formats) + if err != nil { + return nil, err + } + + path, isPath := node.attributes[pathID] + if !isPath { + return nil, newMissingArgumentError(node.name, pathID) + } + + fileWriter, err := NewFileWriter(path) + if err != nil { + return nil, err + } + + return NewFormattedWriter(fileWriter, currentFormat) +} + +// Creates new SMTP writer if encountered in the config file. +func createSMTPWriter(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) { + err := checkUnexpectedAttribute(node, outputFormatID, senderaddressID, senderNameID, hostNameID, hostPortID, userNameID, userPassID, subjectID) + if err != nil { + return nil, err + } + // Node must have children. + if !node.hasChildren() { + return nil, errNodeMustHaveChildren + } + currentFormat, err := getCurrentFormat(node, formatFromParent, formats) + if err != nil { + return nil, err + } + senderAddress, ok := node.attributes[senderaddressID] + if !ok { + return nil, newMissingArgumentError(node.name, senderaddressID) + } + senderName, ok := node.attributes[senderNameID] + if !ok { + return nil, newMissingArgumentError(node.name, senderNameID) + } + // Process child nodes scanning for recipient email addresses and/or CA certificate paths. + var recipientAddresses []string + var caCertDirPaths []string + var mailHeaders []string + for _, childNode := range node.children { + switch childNode.name { + // Extract recipient address from child nodes. + case recipientID: + address, ok := childNode.attributes[addressID] + if !ok { + return nil, newMissingArgumentError(childNode.name, addressID) + } + recipientAddresses = append(recipientAddresses, address) + // Extract CA certificate file path from child nodes. + case cACertDirpathID: + path, ok := childNode.attributes[pathID] + if !ok { + return nil, newMissingArgumentError(childNode.name, pathID) + } + caCertDirPaths = append(caCertDirPaths, path) + + // Extract email headers from child nodes. + case mailHeaderID: + headerName, ok := childNode.attributes[mailHeaderNameID] + if !ok { + return nil, newMissingArgumentError(childNode.name, mailHeaderNameID) + } + + headerValue, ok := childNode.attributes[mailHeaderValueID] + if !ok { + return nil, newMissingArgumentError(childNode.name, mailHeaderValueID) + } + + // Build header line + mailHeaders = append(mailHeaders, fmt.Sprintf("%s: %s", headerName, headerValue)) + default: + return nil, newUnexpectedChildElementError(childNode.name) + } + } + hostName, ok := node.attributes[hostNameID] + if !ok { + return nil, newMissingArgumentError(node.name, hostNameID) + } + + hostPort, ok := node.attributes[hostPortID] + if !ok { + return nil, newMissingArgumentError(node.name, hostPortID) + } + + // Check if the string can really be converted into int. + if _, err := strconv.Atoi(hostPort); err != nil { + return nil, errors.New("invalid host port number") + } + + userName, ok := node.attributes[userNameID] + if !ok { + return nil, newMissingArgumentError(node.name, userNameID) + } + + userPass, ok := node.attributes[userPassID] + if !ok { + return nil, newMissingArgumentError(node.name, userPassID) + } + + // subject is optionally set by configuration. + // default value is defined by DefaultSubjectPhrase constant in the writers_smtpwriter.go + var subjectPhrase = DefaultSubjectPhrase + + subject, ok := node.attributes[subjectID] + if ok { + subjectPhrase = subject + } + + smtpWriter := NewSMTPWriter( + senderAddress, + senderName, + recipientAddresses, + hostName, + hostPort, + userName, + userPass, + caCertDirPaths, + subjectPhrase, + mailHeaders, + ) + + return NewFormattedWriter(smtpWriter, currentFormat) +} + +func createConsoleWriter(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) { + err := checkUnexpectedAttribute(node, outputFormatID) + if err != nil { + return nil, err + } + + if node.hasChildren() { + return nil, errNodeCannotHaveChildren + } + + currentFormat, err := getCurrentFormat(node, formatFromParent, formats) + if err != nil { + return nil, err + } + + consoleWriter, err := NewConsoleWriter() + if err != nil { + return nil, err + } + + return NewFormattedWriter(consoleWriter, currentFormat) +} + +func createconnWriter(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) { + if node.hasChildren() { + return nil, errNodeCannotHaveChildren + } + + err := checkUnexpectedAttribute(node, outputFormatID, connWriterAddrAttr, connWriterNetAttr, connWriterReconnectOnMsgAttr, connWriterUseTLSAttr, connWriterInsecureSkipVerifyAttr) + if err != nil { + return nil, err + } + + currentFormat, err := getCurrentFormat(node, formatFromParent, formats) + if err != nil { + return nil, err + } + + addr, isAddr := node.attributes[connWriterAddrAttr] + if !isAddr { + return nil, newMissingArgumentError(node.name, connWriterAddrAttr) + } + + net, isNet := node.attributes[connWriterNetAttr] + if !isNet { + return nil, newMissingArgumentError(node.name, connWriterNetAttr) + } + + reconnectOnMsg := false + reconnectOnMsgStr, isReconnectOnMsgStr := node.attributes[connWriterReconnectOnMsgAttr] + if isReconnectOnMsgStr { + if reconnectOnMsgStr == "true" { + reconnectOnMsg = true + } else if reconnectOnMsgStr == "false" { + reconnectOnMsg = false + } else { + return nil, errors.New("node '" + node.name + "' has incorrect '" + connWriterReconnectOnMsgAttr + "' attribute value") + } + } + + useTLS := false + useTLSStr, isUseTLSStr := node.attributes[connWriterUseTLSAttr] + if isUseTLSStr { + if useTLSStr == "true" { + useTLS = true + } else if useTLSStr == "false" { + useTLS = false + } else { + return nil, errors.New("node '" + node.name + "' has incorrect '" + connWriterUseTLSAttr + "' attribute value") + } + if useTLS { + insecureSkipVerify := false + insecureSkipVerifyStr, isInsecureSkipVerify := node.attributes[connWriterInsecureSkipVerifyAttr] + if isInsecureSkipVerify { + if insecureSkipVerifyStr == "true" { + insecureSkipVerify = true + } else if insecureSkipVerifyStr == "false" { + insecureSkipVerify = false + } else { + return nil, errors.New("node '" + node.name + "' has incorrect '" + connWriterInsecureSkipVerifyAttr + "' attribute value") + } + } + config := tls.Config{InsecureSkipVerify: insecureSkipVerify} + connWriter := newTLSWriter(net, addr, reconnectOnMsg, &config) + return NewFormattedWriter(connWriter, currentFormat) + } + } + + connWriter := NewConnWriter(net, addr, reconnectOnMsg) + + return NewFormattedWriter(connWriter, currentFormat) +} + +func createRollingFileWriter(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) { + if node.hasChildren() { + return nil, errNodeCannotHaveChildren + } + + rollingTypeStr, isRollingType := node.attributes[rollingFileTypeAttr] + if !isRollingType { + return nil, newMissingArgumentError(node.name, rollingFileTypeAttr) + } + + rollingType, ok := rollingTypeFromString(rollingTypeStr) + if !ok { + return nil, errors.New("unknown rolling file type: " + rollingTypeStr) + } + + currentFormat, err := getCurrentFormat(node, formatFromParent, formats) + if err != nil { + return nil, err + } + + path, isPath := node.attributes[rollingFilePathAttr] + if !isPath { + return nil, newMissingArgumentError(node.name, rollingFilePathAttr) + } + + rollingArchiveStr, archiveAttrExists := node.attributes[rollingFileArchiveAttr] + + var rArchiveType rollingArchiveType + var rArchivePath string + if !archiveAttrExists { + rArchiveType = rollingArchiveNone + rArchivePath = "" + } else { + rArchiveType, ok = rollingArchiveTypeFromString(rollingArchiveStr) + if !ok { + return nil, errors.New("unknown rolling archive type: " + rollingArchiveStr) + } + + if rArchiveType == rollingArchiveNone { + rArchivePath = "" + } else { + rArchivePath, ok = node.attributes[rollingFileArchivePathAttr] + if !ok { + rArchivePath, ok = rollingArchiveTypesDefaultNames[rArchiveType] + if !ok { + return nil, fmt.Errorf("cannot get default filename for archive type = %v", + rArchiveType) + } + } + } + } + + nameMode := rollingNameMode(rollingNameModePostfix) + nameModeStr, ok := node.attributes[rollingFileNameModeAttr] + if ok { + mode, found := rollingNameModeFromString(nameModeStr) + if !found { + return nil, errors.New("unknown rolling filename mode: " + nameModeStr) + } else { + nameMode = mode + } + } + + if rollingType == rollingTypeSize { + err := checkUnexpectedAttribute(node, outputFormatID, rollingFileTypeAttr, rollingFilePathAttr, + rollingFileMaxSizeAttr, rollingFileMaxRollsAttr, rollingFileArchiveAttr, + rollingFileArchivePathAttr, rollingFileNameModeAttr) + if err != nil { + return nil, err + } + + maxSizeStr, ok := node.attributes[rollingFileMaxSizeAttr] + if !ok { + return nil, newMissingArgumentError(node.name, rollingFileMaxSizeAttr) + } + + maxSize, err := strconv.ParseInt(maxSizeStr, 10, 64) + if err != nil { + return nil, err + } + + maxRolls := 0 + maxRollsStr, ok := node.attributes[rollingFileMaxRollsAttr] + if ok { + maxRolls, err = strconv.Atoi(maxRollsStr) + if err != nil { + return nil, err + } + } + + rollingWriter, err := NewRollingFileWriterSize(path, rArchiveType, rArchivePath, maxSize, maxRolls, nameMode) + if err != nil { + return nil, err + } + + return NewFormattedWriter(rollingWriter, currentFormat) + + } else if rollingType == rollingTypeTime { + err := checkUnexpectedAttribute(node, outputFormatID, rollingFileTypeAttr, rollingFilePathAttr, + rollingFileDataPatternAttr, rollingFileArchiveAttr, rollingFileMaxRollsAttr, + rollingFileArchivePathAttr, rollingFileNameModeAttr) + if err != nil { + return nil, err + } + + maxRolls := 0 + maxRollsStr, ok := node.attributes[rollingFileMaxRollsAttr] + if ok { + maxRolls, err = strconv.Atoi(maxRollsStr) + if err != nil { + return nil, err + } + } + + dataPattern, ok := node.attributes[rollingFileDataPatternAttr] + if !ok { + return nil, newMissingArgumentError(node.name, rollingFileDataPatternAttr) + } + + rollingWriter, err := NewRollingFileWriterTime(path, rArchiveType, rArchivePath, maxRolls, dataPattern, rollingIntervalAny, nameMode) + if err != nil { + return nil, err + } + + return NewFormattedWriter(rollingWriter, currentFormat) + } + + return nil, errors.New("incorrect rolling writer type " + rollingTypeStr) +} + +func createbufferedWriter(node *xmlNode, formatFromParent *formatter, formats map[string]*formatter, cfg *CfgParseParams) (interface{}, error) { + err := checkUnexpectedAttribute(node, outputFormatID, bufferedSizeAttr, bufferedFlushPeriodAttr) + if err != nil { + return nil, err + } + + if !node.hasChildren() { + return nil, errNodeMustHaveChildren + } + + currentFormat, err := getCurrentFormat(node, formatFromParent, formats) + if err != nil { + return nil, err + } + + sizeStr, isSize := node.attributes[bufferedSizeAttr] + if !isSize { + return nil, newMissingArgumentError(node.name, bufferedSizeAttr) + } + + size, err := strconv.Atoi(sizeStr) + if err != nil { + return nil, err + } + + flushPeriod := 0 + flushPeriodStr, isFlushPeriod := node.attributes[bufferedFlushPeriodAttr] + if isFlushPeriod { + flushPeriod, err = strconv.Atoi(flushPeriodStr) + if err != nil { + return nil, err + } + } + + // Inner writer couldn't have its own format, so we pass 'currentFormat' as its parent format + receivers, err := createInnerReceivers(node, currentFormat, formats, cfg) + if err != nil { + return nil, err + } + + formattedWriter, ok := receivers[0].(*formattedWriter) + if !ok { + return nil, errors.New("buffered writer's child is not writer") + } + + // ... and then we check that it hasn't changed + if formattedWriter.Format() != currentFormat { + return nil, errors.New("inner writer cannot have his own format") + } + + bufferedWriter, err := NewBufferedWriter(formattedWriter.Writer(), size, time.Duration(flushPeriod)) + if err != nil { + return nil, err + } + + return NewFormattedWriter(bufferedWriter, currentFormat) +} + +// Returns an error if node has any attributes not listed in expectedAttrs. +func checkUnexpectedAttribute(node *xmlNode, expectedAttrs ...string) error { + for attr := range node.attributes { + isExpected := false + for _, expected := range expectedAttrs { + if attr == expected { + isExpected = true + break + } + } + if !isExpected { + return newUnexpectedAttributeError(node.name, attr) + } + } + + return nil +} + +type expectedElementInfo struct { + name string + mandatory bool + multiple bool +} + +func optionalElement(name string) expectedElementInfo { + return expectedElementInfo{name, false, false} +} +func mandatoryElement(name string) expectedElementInfo { + return expectedElementInfo{name, true, false} +} +func multipleElements(name string) expectedElementInfo { + return expectedElementInfo{name, false, true} +} +func multipleMandatoryElements(name string) expectedElementInfo { + return expectedElementInfo{name, true, true} +} + +func checkExpectedElements(node *xmlNode, elements ...expectedElementInfo) error { + for _, element := range elements { + count := 0 + for _, child := range node.children { + if child.name == element.name { + count++ + } + } + + if count == 0 && element.mandatory { + return errors.New(node.name + " does not have mandatory subnode - " + element.name) + } + if count > 1 && !element.multiple { + return errors.New(node.name + " has more then one subnode - " + element.name) + } + } + + for _, child := range node.children { + isExpected := false + for _, element := range elements { + if child.name == element.name { + isExpected = true + } + } + + if !isExpected { + return errors.New(node.name + " has unexpected child: " + child.name) + } + } + + return nil +} diff --git a/lib/seelog/cfg_parser_test.go b/lib/seelog/cfg_parser_test.go new file mode 100644 index 000000000..289da1a71 --- /dev/null +++ b/lib/seelog/cfg_parser_test.go @@ -0,0 +1,1096 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + "testing" +) + +type customTestReceiverOutput struct { + initCalled bool + dataPassed string + messageOutput string + levelOutput LogLevel + closed bool + flushed bool +} +type customTestReceiver struct{ co *customTestReceiverOutput } + +func (cr *customTestReceiver) ReceiveMessage(message string, level LogLevel, context LogContextInterface) error { + cr.co.messageOutput = message + cr.co.levelOutput = level + return nil +} + +func (cr *customTestReceiver) String() string { + return fmt.Sprintf("custom data='%s'", cr.co.dataPassed) +} + +func (cr *customTestReceiver) AfterParse(initArgs CustomReceiverInitArgs) error { + cr.co = new(customTestReceiverOutput) + cr.co.initCalled = true + cr.co.dataPassed = initArgs.XmlCustomAttrs["test"] + return nil +} + +func (cr *customTestReceiver) Flush() { + cr.co.flushed = true +} + +func (cr *customTestReceiver) Close() error { + cr.co.closed = true + return nil +} + +var re = regexp.MustCompile(`[^a-zA-Z0-9]+`) + +func getTestFileName(testName, postfix string) string { + if len(postfix) != 0 { + return strings.ToLower(re.ReplaceAllString(testName, "_")) + "_" + postfix + "_test.log" + } + return strings.ToLower(re.ReplaceAllString(testName, "_")) + "_test.log" +} + +var parserTests []parserTest + +type parserTest struct { + testName string + config string + expected *configForParsing //interface{} + errorExpected bool + parserConfig *CfgParseParams +} + +func getParserTests() []parserTest { + if parserTests == nil { + parserTests = make([]parserTest, 0) + + testName := "Simple file output" + testLogFileName := getTestFileName(testName, "") + testConfig := ` + + + + + + ` + testExpected := new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testfileWriter, _ := NewFileWriter(testLogFileName) + testHeadSplitter, _ := NewSplitDispatcher(DefaultFormatter, []interface{}{testfileWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Filter dispatcher" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testfileWriter, _ = NewFileWriter(testLogFileName) + testFilter, _ := NewFilterDispatcher(DefaultFormatter, []interface{}{testfileWriter}, DebugLvl, InfoLvl, CriticalLvl) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testFilter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Console writer" + testConfig = ` + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testconsoleWriter, _ := NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "SMTP writer" + testConfig = ` + + + + + + + + + + + + ` + + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testSMTPWriter := NewSMTPWriter( + "sa", + "sn", + []string{"ra1", "ra2", "ra3"}, + "hn", + "123", + "un", + "up", + []string{"cacdp1", "cacdp2"}, + DefaultSubjectPhrase, + nil, + ) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testSMTPWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "SMTP writer custom header and subject configuration" + testConfig = ` + + + + + +
+
+
+
+ + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testSMTPWriter = NewSMTPWriter( + "sa", + "sn", + []string{"ra1"}, + "hn", + "123", + "un", + "up", + []string{"cacdp1"}, + "ohlala", + []string{"Priority: Urgent", "Importance: high", "Sensitivity: Company-Confidential", "Auto-Submitted: auto-generated"}, + ) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testSMTPWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Default output" + testConfig = ` + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Asyncloop behavior" + testConfig = ` + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Asynctimer behavior" + testConfig = ` + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = asyncTimerloggerTypeFromString + testExpected.LoggerData = asyncTimerLoggerData{101} + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Rolling file writer size" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testrollingFileWriter, _ := NewRollingFileWriterSize(testLogFileName, rollingArchiveNone, "", 100, 5, rollingNameModePostfix) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Rolling file writer archive zip" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveZip, "log.zip", 100, 5, rollingNameModePostfix) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Rolling file writer archive zip with specified path" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveZip, "test.zip", 100, 5, rollingNameModePrefix) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Rolling file writer archive none" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testrollingFileWriter, _ = NewRollingFileWriterSize(testLogFileName, rollingArchiveNone, "", 100, 5, rollingNameModePostfix) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Rolling file writer date" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testrollingFileWriterTime, _ := NewRollingFileWriterTime(testLogFileName, rollingArchiveNone, "", 0, "2006-01-02T15:04:05Z07:00", rollingIntervalAny, rollingNameModePostfix) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testrollingFileWriterTime}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Buffered writer" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testrollingFileWriterTime, _ = NewRollingFileWriterTime(testLogFileName, rollingArchiveNone, "", 0, "2006-01-02T15:04:05Z07:00", rollingIntervalDaily, rollingNameModePostfix) + testbufferedWriter, _ := NewBufferedWriter(testrollingFileWriterTime, 100500, 100) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testbufferedWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Inner splitter output" + testLogFileName1 := getTestFileName(testName, "1") + testLogFileName2 := getTestFileName(testName, "2") + testLogFileName3 := getTestFileName(testName, "3") + testConfig = ` + + + + + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testfileWriter1, _ := NewFileWriter(testLogFileName2) + testfileWriter2, _ := NewFileWriter(testLogFileName3) + testInnerSplitter, _ := NewSplitDispatcher(DefaultFormatter, []interface{}{testfileWriter1, testfileWriter2}) + testfileWriter, _ = NewFileWriter(testLogFileName1) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testfileWriter, testInnerSplitter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + RegisterReceiver("custom-name-1", &customTestReceiver{}) + + testName = "Custom receiver 1" + testConfig = ` + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testCustomReceiver, _ := NewCustomReceiverDispatcher(DefaultFormatter, "custom-name-1", CustomReceiverInitArgs{ + XmlCustomAttrs: map[string]string{ + "test": "set", + }, + }) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testCustomReceiver}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Custom receiver 2" + testConfig = ` + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + crec := &customTestReceiver{} + cargs := CustomReceiverInitArgs{ + XmlCustomAttrs: map[string]string{ + "test": "set2", + }, + } + crec.AfterParse(cargs) + testCustomReceiver2, _ := NewCustomReceiverDispatcherByValue(DefaultFormatter, crec, "custom-name-2", cargs) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testCustomReceiver2}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + fnc := func(initArgs CustomReceiverInitArgs) (CustomReceiver, error) { + return &customTestReceiver{}, nil + } + cfg := CfgParseParams{ + CustomReceiverProducers: map[string]CustomReceiverProducer{ + "custom-name-2": CustomReceiverProducer(fnc), + }, + } + testExpected.Params = &cfg + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, &cfg}) + + RegisterReceiver("-", &customTestReceiver{}) + testName = "Custom receiver 3" + testConfig = ` + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + creccustom := &customTestReceiver{} + cargs3 := CustomReceiverInitArgs{ + XmlCustomAttrs: map[string]string{ + "test": "set3", + }, + } + creccustom.AfterParse(cargs3) + testCustomReceiver, _ = NewCustomReceiverDispatcherByValue(DefaultFormatter, creccustom, "-", cargs3) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testCustomReceiver}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Custom receivers with formats" + testConfig = ` + + + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testCustomReceivers := make([]*customReceiverDispatcher, 3) + for i := 0; i < 3; i++ { + testCustomReceivers[i], _ = NewCustomReceiverDispatcher(DefaultFormatter, "custom-name-1", CustomReceiverInitArgs{ + XmlCustomAttrs: map[string]string{ + "test": fmt.Sprintf("set%d", i+1), + }, + }) + } + + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testCustomReceivers[0], testCustomReceivers[1], testCustomReceivers[2]}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Format" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testfileWriter, _ = NewFileWriter(testLogFileName) + testFormat, _ := NewFormatter("%Level %Msg %File") + testHeadSplitter, _ = NewSplitDispatcher(testFormat, []interface{}{testfileWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Format2" + testLogFileName = getTestFileName(testName, "") + testLogFileName1 = getTestFileName(testName, "1") + testConfig = ` + + + + + + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testfileWriter, _ = NewFileWriter(testLogFileName) + testfileWriter1, _ = NewFileWriter(testLogFileName1) + testFormat1, _ := NewFormatter("%Level %Msg %File") + testFormat2, _ := NewFormatter("%l %Msg") + formattedWriter, _ := NewFormattedWriter(testfileWriter1, testFormat2) + testHeadSplitter, _ = NewSplitDispatcher(testFormat1, []interface{}{testfileWriter, formattedWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Minlevel = warn" + testConfig = `` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(WarnLvl, CriticalLvl) + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Maxlevel = trace" + testConfig = `` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, TraceLvl) + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Level between info and error" + testConfig = `` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(InfoLvl, ErrorLvl) + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Off with minlevel" + testConfig = `` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewOffConstraints() + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Off with levels" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Levels list" + testConfig = `` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewListConstraints([]LogLevel{ + DebugLvl, InfoLvl, CriticalLvl}) + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Errors #1" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #2" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #3" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #4" + testConfig = `` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, Off) + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = asyncLooploggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Errors #5" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #6" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #7" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #8" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #9" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #10" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #11" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #12" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #13" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #14" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #15" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #16" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #17" + testLogFileName = getTestFileName(testName, "") + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #18" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #19" + testConfig = `` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Exceptions: restricting" + testConfig = + ` + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + listConstraint, _ := NewOffConstraints() + exception, _ := NewLogLevelException("Test*", "someFile.go", listConstraint) + testExpected.Exceptions = []*LogLevelException{exception} + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Exceptions: allowing #1" + testConfig = + ` + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewListConstraints([]LogLevel{ErrorLvl}) + minMaxConstraint, _ := NewMinMaxConstraints(TraceLvl, CriticalLvl) + exception, _ = NewLogLevelException("*", "testfile.go", minMaxConstraint) + testExpected.Exceptions = []*LogLevelException{exception} + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Exceptions: allowing #2" + testConfig = ` + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewOffConstraints() + minMaxConstraint, _ = NewMinMaxConstraints(WarnLvl, CriticalLvl) + exception, _ = NewLogLevelException("*", "testfile.go", minMaxConstraint) + testExpected.Exceptions = []*LogLevelException{exception} + testconsoleWriter, _ = NewConsoleWriter() + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testconsoleWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Predefined formats" + formatID := predefinedPrefix + "xml-debug-short" + testConfig = ` + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testconsoleWriter, _ = NewConsoleWriter() + testFormat, _ = predefinedFormats[formatID] + testHeadSplitter, _ = NewSplitDispatcher(testFormat, []interface{}{testconsoleWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Predefined formats redefine" + testLogFileName = getTestFileName(testName, "") + formatID = predefinedPrefix + "xml-debug-short" + testConfig = ` + + + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testfileWriter, _ = NewFileWriter(testLogFileName) + testFormat, _ = NewFormatter("%Level %Msg %File") + testHeadSplitter, _ = NewSplitDispatcher(testFormat, []interface{}{testfileWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Conn writer 1" + testConfig = ` + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testConnWriter := NewConnWriter("tcp", ":8888", false) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testConnWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Conn writer 2" + testConfig = ` + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testConnWriter = NewConnWriter("tcp", ":8888", true) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{testConnWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + testName = "Errors #11" + testConfig = ` + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #12" + testConfig = ` + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #13" + testConfig = ` + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #14" + testConfig = ` + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #15" + testConfig = ` + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #16" + testConfig = ` + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #17" + testConfig = ` + + + + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #18" + testConfig = ` + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #19" + testConfig = ` + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #20" + testConfig = ` + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #21" + testConfig = ` + + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #22" + testConfig = ` + + + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #23" + testConfig = ` + + + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #24" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #25" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + + + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Errors #26" + testConfig = ` + + + + + ` + parserTests = append(parserTests, parserTest{testName, testConfig, nil, true, nil}) + + testName = "Buffered writer same formatid override" + testLogFileName = getTestFileName(testName, "") + testConfig = ` + + + + + + + + + + ` + testExpected = new(configForParsing) + testExpected.Constraints, _ = NewMinMaxConstraints(TraceLvl, CriticalLvl) + testExpected.Exceptions = nil + testrollingFileWriterTime, _ = NewRollingFileWriterTime(testLogFileName, rollingArchiveNone, "", 0, "2006-01-02T15:04:05Z07:00", rollingIntervalDaily, rollingNameModePrefix) + testbufferedWriter, _ = NewBufferedWriter(testrollingFileWriterTime, 100500, 100) + testFormat, _ = NewFormatter("%Level %Msg %File 123") + formattedWriter, _ = NewFormattedWriter(testbufferedWriter, testFormat) + testHeadSplitter, _ = NewSplitDispatcher(DefaultFormatter, []interface{}{formattedWriter}) + testExpected.LogType = syncloggerTypeFromString + testExpected.RootDispatcher = testHeadSplitter + parserTests = append(parserTests, parserTest{testName, testConfig, testExpected, false, nil}) + + } + + return parserTests +} + +// Temporary solution: compare by string identity. Not the best solution in +// terms of performance, but a valid one in terms of comparison, because +// every seelog dispatcher/receiver must have a valid String() func +// that fully represents its internal parameters. +func configsAreEqual(conf1 *configForParsing, conf2 interface{}) bool { + if conf1 == nil { + return conf2 == nil + } + if conf2 == nil { + return conf1 == nil + } + + // configForParsing, ok := conf2 //.(*configForParsing) + // if !ok { + // return false + // } + + return fmt.Sprintf("%v", conf1) == fmt.Sprintf("%v", conf2) //configForParsing) +} + +func testLogFileFilter(fn string) bool { + return ".log" == filepath.Ext(fn) +} + +func cleanupAfterCfgTest(t *testing.T) { + toDel, err := getDirFilePaths(".", testLogFileFilter, true) + if nil != err { + t.Fatal("Cannot list files in test directory!") + } + + for _, p := range toDel { + err = tryRemoveFile(p) + if nil != err { + t.Errorf("cannot remove file %s in test directory: %s", p, err.Error()) + } + } +} + +func parseTest(test parserTest, t *testing.T) { + conf, err := configFromReaderWithConfig(strings.NewReader(test.config), test.parserConfig) + if /*err != nil &&*/ conf != nil && conf.RootDispatcher != nil { + defer func() { + if err = conf.RootDispatcher.Close(); err != nil { + t.Errorf("\n----ERROR while closing root dispatcher in %s test: %s", test.testName, err) + } + }() + } + + if (err != nil) != test.errorExpected { + t.Errorf("\n----ERROR in %s:\nConfig: %s\n* Expected error:%t. Got error: %t\n", + test.testName, test.config, test.errorExpected, (err != nil)) + if err != nil { + t.Logf("%s\n", err.Error()) + } + return + } + + if err == nil && !configsAreEqual(conf, test.expected) { + t.Errorf("\n----ERROR in %s:\nConfig: %s\n* Expected: %v. \n* Got: %v\n", + test.testName, test.config, test.expected, conf) + } +} + +func TestParser(t *testing.T) { + defer cleanupAfterCfgTest(t) + + for _, test := range getParserTests() { + parseTest(test, t) + } +} diff --git a/lib/seelog/common_closer.go b/lib/seelog/common_closer.go new file mode 100644 index 000000000..1319c2217 --- /dev/null +++ b/lib/seelog/common_closer.go @@ -0,0 +1,25 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog diff --git a/lib/seelog/common_constraints.go b/lib/seelog/common_constraints.go new file mode 100644 index 000000000..126d0ec02 --- /dev/null +++ b/lib/seelog/common_constraints.go @@ -0,0 +1,162 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "fmt" + "strings" +) + +// Represents constraints which form a general rule for log levels selection +type logLevelConstraints interface { + IsAllowed(level LogLevel) bool +} + +// A minMaxConstraints represents constraints which use minimal and maximal allowed log levels. +type minMaxConstraints struct { + min LogLevel + max LogLevel +} + +// NewMinMaxConstraints creates a new minMaxConstraints struct with the specified min and max levels. +func NewMinMaxConstraints(min LogLevel, max LogLevel) (*minMaxConstraints, error) { + if min > max { + return nil, fmt.Errorf("min level can't be greater than max. Got min: %d, max: %d", min, max) + } + if min < TraceLvl || min > Off { + return nil, fmt.Errorf("min level can't be less than Trace or greater than Off. Got min: %d", min) + } + if max < TraceLvl || max > Off { + return nil, fmt.Errorf("max level can't be less than Trace or greater than Off. Got max: %d", max) + } + + return &minMaxConstraints{min, max}, nil +} + +// IsAllowed returns true, if log level is in [min, max] range (inclusive). +func (minMaxConstr *minMaxConstraints) IsAllowed(level LogLevel) bool { + return level >= minMaxConstr.min && level <= minMaxConstr.max +} + +func (minMaxConstr *minMaxConstraints) String() string { + return fmt.Sprintf("Min: %s. Max: %s", minMaxConstr.min, minMaxConstr.max) +} + +//======================================================= + +// A listConstraints represents constraints which use allowed log levels list. +type listConstraints struct { + allowedLevels map[LogLevel]bool +} + +// NewListConstraints creates a new listConstraints struct with the specified allowed levels. +func NewListConstraints(allowList []LogLevel) (*listConstraints, error) { + if allowList == nil { + return nil, errors.New("list can't be nil") + } + + allowLevels, err := createMapFromList(allowList) + if err != nil { + return nil, err + } + err = validateOffLevel(allowLevels) + if err != nil { + return nil, err + } + + return &listConstraints{allowLevels}, nil +} + +func (listConstr *listConstraints) String() string { + allowedList := "List: " + + listLevel := make([]string, len(listConstr.allowedLevels)) + + var logLevel LogLevel + i := 0 + for logLevel = TraceLvl; logLevel <= Off; logLevel++ { + if listConstr.allowedLevels[logLevel] { + listLevel[i] = logLevel.String() + i++ + } + } + + allowedList += strings.Join(listLevel, ",") + + return allowedList +} + +func createMapFromList(allowedList []LogLevel) (map[LogLevel]bool, error) { + allowedLevels := make(map[LogLevel]bool, 0) + for _, level := range allowedList { + if level < TraceLvl || level > Off { + return nil, fmt.Errorf("level can't be less than Trace or greater than Critical. Got level: %d", level) + } + allowedLevels[level] = true + } + return allowedLevels, nil +} +func validateOffLevel(allowedLevels map[LogLevel]bool) error { + if _, ok := allowedLevels[Off]; ok && len(allowedLevels) > 1 { + return errors.New("logLevel Off cant be mixed with other levels") + } + + return nil +} + +// IsAllowed returns true, if log level is in allowed log levels list. +// If the list contains the only item 'common.Off' then IsAllowed will always return false for any input values. +func (listConstr *listConstraints) IsAllowed(level LogLevel) bool { + for l := range listConstr.allowedLevels { + if l == level && level != Off { + return true + } + } + + return false +} + +// AllowedLevels returns allowed levels configuration as a map. +func (listConstr *listConstraints) AllowedLevels() map[LogLevel]bool { + return listConstr.allowedLevels +} + +//======================================================= + +type offConstraints struct { +} + +func NewOffConstraints() (*offConstraints, error) { + return &offConstraints{}, nil +} + +func (offConstr *offConstraints) IsAllowed(level LogLevel) bool { + return false +} + +func (offConstr *offConstraints) String() string { + return "Off constraint" +} diff --git a/lib/seelog/common_constraints_test.go b/lib/seelog/common_constraints_test.go new file mode 100644 index 000000000..bb9918e64 --- /dev/null +++ b/lib/seelog/common_constraints_test.go @@ -0,0 +1,196 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "testing" +) + +func TestInvalidminMaxConstraints(t *testing.T) { + constr, err := NewMinMaxConstraints(CriticalLvl, WarnLvl) + + if err == nil || constr != nil { + t.Errorf("expected an error and a nil value for minmax constraints: min = %d, max = %d. Got: %v, %v", + CriticalLvl, WarnLvl, err, constr) + return + } +} + +func TestInvalidLogLevels(t *testing.T) { + var invalidMin uint8 = 123 + var invalidMax uint8 = 124 + minMaxConstr, errMinMax := NewMinMaxConstraints(LogLevel(invalidMin), LogLevel(invalidMax)) + + if errMinMax == nil || minMaxConstr != nil { + t.Errorf("expected an error and a nil value for minmax constraints: min = %d, max = %d. Got: %v, %v", + invalidMin, invalidMax, errMinMax, minMaxConstr) + return + } + + invalidList := []LogLevel{145} + + listConstr, errList := NewListConstraints(invalidList) + + if errList == nil || listConstr != nil { + t.Errorf("expected an error and a nil value for constraints list: %v. Got: %v, %v", + invalidList, errList, listConstr) + return + } +} + +func TestlistConstraintsWithDuplicates(t *testing.T) { + duplicateList := []LogLevel{TraceLvl, DebugLvl, InfoLvl, + WarnLvl, ErrorLvl, CriticalLvl, CriticalLvl, CriticalLvl} + + listConstr, errList := NewListConstraints(duplicateList) + + if errList != nil || listConstr == nil { + t.Errorf("expected a valid constraints list struct for: %v, got error: %v, value: %v", + duplicateList, errList, listConstr) + return + } + + listLevels := listConstr.AllowedLevels() + + if listLevels == nil { + t.Fatalf("listConstr.AllowedLevels() == nil") + return + } + + if len(listLevels) != 6 { + t.Errorf("expected: listConstr.AllowedLevels() length == 6. Got: %d", len(listLevels)) + return + } +} + +func TestlistConstraintsWithOffInList(t *testing.T) { + offList := []LogLevel{TraceLvl, DebugLvl, Off} + + listConstr, errList := NewListConstraints(offList) + + if errList == nil || listConstr != nil { + t.Errorf("expected an error and a nil value for constraints list with 'Off': %v. Got: %v, %v", + offList, errList, listConstr) + return + } +} + +type logLevelTestCase struct { + level LogLevel + allowed bool +} + +var minMaxTests = []logLevelTestCase{ + {TraceLvl, false}, + {DebugLvl, false}, + {InfoLvl, true}, + {WarnLvl, true}, + {ErrorLvl, false}, + {CriticalLvl, false}, + {123, false}, + {6, false}, +} + +func TestValidminMaxConstraints(t *testing.T) { + + constr, err := NewMinMaxConstraints(InfoLvl, WarnLvl) + + if err != nil || constr == nil { + t.Errorf("expected a valid constraints struct for minmax constraints: min = %d, max = %d. Got: %v, %v", + InfoLvl, WarnLvl, err, constr) + return + } + + for _, minMaxTest := range minMaxTests { + allowed := constr.IsAllowed(minMaxTest.level) + if allowed != minMaxTest.allowed { + t.Errorf("expected IsAllowed() = %t for level = %d. Got: %t", + minMaxTest.allowed, minMaxTest.level, allowed) + return + } + } +} + +var listTests = []logLevelTestCase{ + {TraceLvl, true}, + {DebugLvl, false}, + {InfoLvl, true}, + {WarnLvl, true}, + {ErrorLvl, false}, + {CriticalLvl, true}, + {123, false}, + {6, false}, +} + +func TestValidlistConstraints(t *testing.T) { + validList := []LogLevel{TraceLvl, InfoLvl, WarnLvl, CriticalLvl} + constr, err := NewListConstraints(validList) + + if err != nil || constr == nil { + t.Errorf("expected a valid constraints list struct for: %v. Got error: %v, value: %v", + validList, err, constr) + return + } + + for _, minMaxTest := range listTests { + allowed := constr.IsAllowed(minMaxTest.level) + if allowed != minMaxTest.allowed { + t.Errorf("expected IsAllowed() = %t for level = %d. Got: %t", + minMaxTest.allowed, minMaxTest.level, allowed) + return + } + } +} + +var offTests = []logLevelTestCase{ + {TraceLvl, false}, + {DebugLvl, false}, + {InfoLvl, false}, + {WarnLvl, false}, + {ErrorLvl, false}, + {CriticalLvl, false}, + {123, false}, + {6, false}, +} + +func TestValidListoffConstraints(t *testing.T) { + validList := []LogLevel{Off} + constr, err := NewListConstraints(validList) + + if err != nil || constr == nil { + t.Errorf("expected a valid constraints list struct for: %v. Got error: %v, value: %v", + validList, err, constr) + return + } + + for _, minMaxTest := range offTests { + allowed := constr.IsAllowed(minMaxTest.level) + if allowed != minMaxTest.allowed { + t.Errorf("expected IsAllowed() = %t for level = %d. Got: %t", + minMaxTest.allowed, minMaxTest.level, allowed) + return + } + } +} diff --git a/lib/seelog/common_context.go b/lib/seelog/common_context.go new file mode 100644 index 000000000..04bc2235e --- /dev/null +++ b/lib/seelog/common_context.go @@ -0,0 +1,194 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "time" +) + +var workingDir = "/" + +func init() { + wd, err := os.Getwd() + if err == nil { + workingDir = filepath.ToSlash(wd) + "/" + } +} + +// Represents runtime caller context. +type LogContextInterface interface { + // Caller's function name. + Func() string + // Caller's line number. + Line() int + // Caller's file short path (in slashed form). + ShortPath() string + // Caller's file full path (in slashed form). + FullPath() string + // Caller's file name (without path). + FileName() string + // True if the context is correct and may be used. + // If false, then an error in context evaluation occurred and + // all its other data may be corrupted. + IsValid() bool + // Time when log function was called. + CallTime() time.Time + // Custom context that can be set by calling logger.SetContext + CustomContext() interface{} +} + +// Returns context of the caller +func currentContext(custom interface{}) (LogContextInterface, error) { + return specifyContext(1, custom) +} + +func extractCallerInfo(skip int) (fullPath string, shortPath string, funcName string, line int, err error) { + pc, fp, ln, ok := runtime.Caller(skip) + if !ok { + err = fmt.Errorf("error during runtime.Caller") + return + } + line = ln + fullPath = fp + if strings.HasPrefix(fp, workingDir) { + shortPath = fp[len(workingDir):] + } else { + shortPath = fp + } + funcName = runtime.FuncForPC(pc).Name() + if strings.HasPrefix(funcName, workingDir) { + funcName = funcName[len(workingDir):] + } + return +} + +// Returns context of the function with placed "skip" stack frames of the caller +// If skip == 0 then behaves like currentContext +// Context is returned in any situation, even if error occurs. But, if an error +// occurs, the returned context is an error context, which contains no paths +// or names, but states that they can't be extracted. +func specifyContext(skip int, custom interface{}) (LogContextInterface, error) { + callTime := time.Now() + if skip < 0 { + err := fmt.Errorf("can not skip negative stack frames") + return &errorContext{callTime, err}, err + } + fullPath, shortPath, funcName, line, err := extractCallerInfo(skip + 2) + if err != nil { + return &errorContext{callTime, err}, err + } + _, fileName := filepath.Split(fullPath) + return &logContext{funcName, line, shortPath, fullPath, fileName, callTime, custom}, nil +} + +// Represents a normal runtime caller context. +type logContext struct { + funcName string + line int + shortPath string + fullPath string + fileName string + callTime time.Time + custom interface{} +} + +func (context *logContext) IsValid() bool { + return true +} + +func (context *logContext) Func() string { + return context.funcName +} + +func (context *logContext) Line() int { + return context.line +} + +func (context *logContext) ShortPath() string { + return context.shortPath +} + +func (context *logContext) FullPath() string { + return context.fullPath +} + +func (context *logContext) FileName() string { + return context.fileName +} + +func (context *logContext) CallTime() time.Time { + return context.callTime +} + +func (context *logContext) CustomContext() interface{} { + return context.custom +} + +// Represents an error context +type errorContext struct { + errorTime time.Time + err error +} + +func (errContext *errorContext) getErrorText(prefix string) string { + return fmt.Sprintf("%s() error: %s", prefix, errContext.err) +} + +func (errContext *errorContext) IsValid() bool { + return false +} + +func (errContext *errorContext) Line() int { + return -1 +} + +func (errContext *errorContext) Func() string { + return errContext.getErrorText("Func") +} + +func (errContext *errorContext) ShortPath() string { + return errContext.getErrorText("ShortPath") +} + +func (errContext *errorContext) FullPath() string { + return errContext.getErrorText("FullPath") +} + +func (errContext *errorContext) FileName() string { + return errContext.getErrorText("FileName") +} + +func (errContext *errorContext) CallTime() time.Time { + return errContext.errorTime +} + +func (errContext *errorContext) CustomContext() interface{} { + return nil +} diff --git a/lib/seelog/common_context_test.go b/lib/seelog/common_context_test.go new file mode 100644 index 000000000..bd1e47aba --- /dev/null +++ b/lib/seelog/common_context_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const ( + testShortPath = "common_context_test.go" +) + +var ( + commonPrefix string + testFullPath string +) + +func init() { + // Here we remove the hardcoding of the package name which + // may break forks and some CI environments such as jenkins. + _, _, funcName, _, _ := extractCallerInfo(1) + preIndex := strings.Index(funcName, "init·") + if preIndex == -1 { + preIndex = strings.Index(funcName, "init") + } + commonPrefix = funcName[:preIndex] + wd, err := os.Getwd() + if err == nil { + // Transform the file path into a slashed form: + // This is the proper platform-neutral way. + testFullPath = filepath.ToSlash(filepath.Join(wd, testShortPath)) + } +} + +func TestContext(t *testing.T) { + context, err := currentContext(nil) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if context == nil { + t.Fatalf("unexpected error: context is nil") + } + if fn, funcName := context.Func(), commonPrefix+"TestContext"; fn != funcName { + // Account for a case when the func full path is longer than commonPrefix but includes it. + if !strings.HasSuffix(fn, funcName) { + t.Errorf("expected context.Func == %s ; got %s", funcName, context.Func()) + } + } + if context.ShortPath() != testShortPath { + t.Errorf("expected context.ShortPath == %s ; got %s", testShortPath, context.ShortPath()) + } + if len(testFullPath) == 0 { + t.Fatal("working directory seems invalid") + } + if context.FullPath() != testFullPath { + t.Errorf("expected context.FullPath == %s ; got %s", testFullPath, context.FullPath()) + } +} + +func innerContext() (context LogContextInterface, err error) { + return currentContext(nil) +} + +func TestInnerContext(t *testing.T) { + context, err := innerContext() + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if context == nil { + t.Fatalf("unexpected error: context is nil") + } + if fn, funcName := context.Func(), commonPrefix+"innerContext"; fn != funcName { + // Account for a case when the func full path is longer than commonPrefix but includes it. + if !strings.HasSuffix(fn, funcName) { + t.Errorf("expected context.Func == %s ; got %s", funcName, context.Func()) + } + } + if context.ShortPath() != testShortPath { + t.Errorf("expected context.ShortPath == %s ; got %s", testShortPath, context.ShortPath()) + } + if len(testFullPath) == 0 { + t.Fatal("working directory seems invalid") + } + if context.FullPath() != testFullPath { + t.Errorf("expected context.FullPath == %s ; got %s", testFullPath, context.FullPath()) + } +} + +type testContext struct { + field string +} + +func TestCustomContext(t *testing.T) { + expected := "testStr" + context, err := currentContext(&testContext{expected}) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + if st, _ := context.CustomContext().(*testContext); st.field != expected { + t.Errorf("expected context.CustomContext == %s ; got %s", expected, st.field) + } +} diff --git a/lib/seelog/common_exception.go b/lib/seelog/common_exception.go new file mode 100644 index 000000000..9acc27507 --- /dev/null +++ b/lib/seelog/common_exception.go @@ -0,0 +1,194 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +// Used in rules creation to validate input file and func filters +var ( + fileFormatValidator = regexp.MustCompile(`[a-zA-Z0-9\\/ _\*\.]*`) + funcFormatValidator = regexp.MustCompile(`[a-zA-Z0-9_\*\.]*`) +) + +// LogLevelException represents an exceptional case used when you need some specific files or funcs to +// override general constraints and to use their own. +type LogLevelException struct { + funcPatternParts []string + filePatternParts []string + + funcPattern string + filePattern string + + constraints logLevelConstraints +} + +// NewLogLevelException creates a new exception. +func NewLogLevelException(funcPattern string, filePattern string, constraints logLevelConstraints) (*LogLevelException, error) { + if constraints == nil { + return nil, errors.New("constraints can not be nil") + } + + exception := new(LogLevelException) + + err := exception.initFuncPatternParts(funcPattern) + if err != nil { + return nil, err + } + exception.funcPattern = strings.Join(exception.funcPatternParts, "") + + err = exception.initFilePatternParts(filePattern) + if err != nil { + return nil, err + } + exception.filePattern = strings.Join(exception.filePatternParts, "") + + exception.constraints = constraints + + return exception, nil +} + +// MatchesContext returns true if context matches the patterns of this LogLevelException +func (logLevelEx *LogLevelException) MatchesContext(context LogContextInterface) bool { + return logLevelEx.match(context.Func(), context.FullPath()) +} + +// IsAllowed returns true if log level is allowed according to the constraints of this LogLevelException +func (logLevelEx *LogLevelException) IsAllowed(level LogLevel) bool { + return logLevelEx.constraints.IsAllowed(level) +} + +// FuncPattern returns the function pattern of a exception +func (logLevelEx *LogLevelException) FuncPattern() string { + return logLevelEx.funcPattern +} + +// FuncPattern returns the file pattern of a exception +func (logLevelEx *LogLevelException) FilePattern() string { + return logLevelEx.filePattern +} + +// initFuncPatternParts checks whether the func filter has a correct format and splits funcPattern on parts +func (logLevelEx *LogLevelException) initFuncPatternParts(funcPattern string) (err error) { + + if funcFormatValidator.FindString(funcPattern) != funcPattern { + return errors.New("func path \"" + funcPattern + "\" contains incorrect symbols. Only a-z A-Z 0-9 _ * . allowed)") + } + + logLevelEx.funcPatternParts = splitPattern(funcPattern) + return nil +} + +// Checks whether the file filter has a correct format and splits file patterns using splitPattern. +func (logLevelEx *LogLevelException) initFilePatternParts(filePattern string) (err error) { + + if fileFormatValidator.FindString(filePattern) != filePattern { + return errors.New("file path \"" + filePattern + "\" contains incorrect symbols. Only a-z A-Z 0-9 \\ / _ * . allowed)") + } + + logLevelEx.filePatternParts = splitPattern(filePattern) + return err +} + +func (logLevelEx *LogLevelException) match(funcPath string, filePath string) bool { + if !stringMatchesPattern(logLevelEx.funcPatternParts, funcPath) { + return false + } + return stringMatchesPattern(logLevelEx.filePatternParts, filePath) +} + +func (logLevelEx *LogLevelException) String() string { + str := fmt.Sprintf("Func: %s File: %s", logLevelEx.funcPattern, logLevelEx.filePattern) + + if logLevelEx.constraints != nil { + str += fmt.Sprintf("Constr: %s", logLevelEx.constraints) + } else { + str += "nil" + } + + return str +} + +// splitPattern splits pattern into strings and asterisks. Example: "ab*cde**f" -> ["ab", "*", "cde", "*", "f"] +func splitPattern(pattern string) []string { + var patternParts []string + var lastChar rune + for _, char := range pattern { + if char == '*' { + if lastChar != '*' { + patternParts = append(patternParts, "*") + } + } else { + if len(patternParts) != 0 && lastChar != '*' { + patternParts[len(patternParts)-1] += string(char) + } else { + patternParts = append(patternParts, string(char)) + } + } + lastChar = char + } + + return patternParts +} + +// stringMatchesPattern check whether testString matches pattern with asterisks. +// Standard regexp functionality is not used here because of performance issues. +func stringMatchesPattern(patternparts []string, testString string) bool { + if len(patternparts) == 0 { + return len(testString) == 0 + } + + part := patternparts[0] + if part != "*" { + index := strings.Index(testString, part) + if index == 0 { + return stringMatchesPattern(patternparts[1:], testString[len(part):]) + } + } else { + if len(patternparts) == 1 { + return true + } + + newTestString := testString + part = patternparts[1] + for { + index := strings.Index(newTestString, part) + if index == -1 { + break + } + + newTestString = newTestString[index+len(part):] + result := stringMatchesPattern(patternparts[2:], newTestString) + if result { + return true + } + } + } + return false +} diff --git a/lib/seelog/common_exception_test.go b/lib/seelog/common_exception_test.go new file mode 100644 index 000000000..d98c28034 --- /dev/null +++ b/lib/seelog/common_exception_test.go @@ -0,0 +1,98 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "testing" +) + +type exceptionTestCase struct { + funcPattern string + filePattern string + funcName string + fileName string + match bool +} + +var exceptionTestCases = []exceptionTestCase{ + {"*", "*", "func", "file", true}, + {"func*", "*", "func", "file", true}, + {"*func", "*", "func", "file", true}, + {"*func", "*", "1func", "file", true}, + {"func*", "*", "func1", "file", true}, + {"fu*nc", "*", "func", "file", true}, + {"fu*nc", "*", "fu1nc", "file", true}, + {"fu*nc", "*", "func1nc", "file", true}, + {"*fu*nc*", "*", "somefuntonc", "file", true}, + {"fu*nc", "*", "f1nc", "file", false}, + {"func*", "*", "fun", "file", false}, + {"fu*nc", "*", "func1n", "file", false}, + {"**f**u**n**c**", "*", "func1n", "file", true}, +} + +func TestMatchingCorrectness(t *testing.T) { + constraints, err := NewListConstraints([]LogLevel{TraceLvl}) + if err != nil { + t.Error(err) + return + } + + for _, testCase := range exceptionTestCases { + rule, ruleError := NewLogLevelException(testCase.funcPattern, testCase.filePattern, constraints) + if ruleError != nil { + t.Fatalf("Unexpected error on rule creation: [ %v, %v ]. %v", + testCase.funcPattern, testCase.filePattern, ruleError) + } + + match := rule.match(testCase.funcName, testCase.fileName) + if match != testCase.match { + t.Errorf("incorrect matching for [ %v, %v ] [ %v, %v ] Expected: %t. Got: %t", + testCase.funcPattern, testCase.filePattern, testCase.funcName, testCase.fileName, testCase.match, match) + } + } +} + +func TestAsterisksReducing(t *testing.T) { + constraints, err := NewListConstraints([]LogLevel{TraceLvl}) + if err != nil { + t.Error(err) + return + } + + rule, err := NewLogLevelException("***func**", "fi*****le", constraints) + if err != nil { + t.Error(err) + return + } + expectFunc := "*func*" + if rule.FuncPattern() != expectFunc { + t.Errorf("asterisks must be reduced. Expect:%v, Got:%v", expectFunc, rule.FuncPattern()) + } + + expectFile := "fi*le" + if rule.FilePattern() != expectFile { + t.Errorf("asterisks must be reduced. Expect:%v, Got:%v", expectFile, rule.FilePattern()) + } +} diff --git a/lib/seelog/common_flusher.go b/lib/seelog/common_flusher.go new file mode 100644 index 000000000..0ef077c8d --- /dev/null +++ b/lib/seelog/common_flusher.go @@ -0,0 +1,31 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +// flusherInterface represents all objects that have to do cleanup +// at certain moments of time (e.g. before app shutdown to avoid data loss) +type flusherInterface interface { + Flush() +} diff --git a/lib/seelog/common_loglevel.go b/lib/seelog/common_loglevel.go new file mode 100644 index 000000000..d54ecf270 --- /dev/null +++ b/lib/seelog/common_loglevel.go @@ -0,0 +1,81 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +// Log level type +type LogLevel uint8 + +// Log levels +const ( + TraceLvl = iota + DebugLvl + InfoLvl + WarnLvl + ErrorLvl + CriticalLvl + Off +) + +// Log level string representations (used in configuration files) +const ( + TraceStr = "trace" + DebugStr = "debug" + InfoStr = "info" + WarnStr = "warn" + ErrorStr = "error" + CriticalStr = "critical" + OffStr = "off" +) + +var levelToStringRepresentations = map[LogLevel]string{ + TraceLvl: TraceStr, + DebugLvl: DebugStr, + InfoLvl: InfoStr, + WarnLvl: WarnStr, + ErrorLvl: ErrorStr, + CriticalLvl: CriticalStr, + Off: OffStr, +} + +// LogLevelFromString parses a string and returns a corresponding log level, if sucessfull. +func LogLevelFromString(levelStr string) (level LogLevel, found bool) { + for lvl, lvlStr := range levelToStringRepresentations { + if lvlStr == levelStr { + return lvl, true + } + } + + return 0, false +} + +// LogLevelToString returns seelog string representation for a specified level. Returns "" for invalid log levels. +func (level LogLevel) String() string { + levelStr, ok := levelToStringRepresentations[level] + if ok { + return levelStr + } + + return "" +} diff --git a/lib/seelog/dispatch_custom.go b/lib/seelog/dispatch_custom.go new file mode 100644 index 000000000..17d518e41 --- /dev/null +++ b/lib/seelog/dispatch_custom.go @@ -0,0 +1,243 @@ +// Copyright (c) 2013 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "fmt" + "reflect" + "sort" +) + +var registeredReceivers = make(map[string]reflect.Type) + +// RegisterReceiver records a custom receiver type, identified by a value +// of that type (second argument), under the specified name. Registered +// names can be used in the "name" attribute of config items. +// +// RegisterReceiver takes the type of the receiver argument, without taking +// the value into the account. So do NOT enter any data to the second argument +// and only call it like: +// +// RegisterReceiver("somename", &MyReceiverType{}) +// +// After that, when a '' config tag with this name is used, +// a receiver of the specified type would be instantiated. Check +// CustomReceiver comments for interface details. +// +// NOTE 1: RegisterReceiver fails if you attempt to register different types +// with the same name. +// +// NOTE 2: RegisterReceiver registers those receivers that must be used in +// the configuration files ( items). Basically it is just the way +// you tell seelog config parser what should it do when it meets a +// tag with a specific name and data attributes. +// +// But If you are only using seelog as a proxy to an already instantiated +// CustomReceiver (via LoggerFromCustomReceiver func), you should not call RegisterReceiver. +func RegisterReceiver(name string, receiver CustomReceiver) { + newType := reflect.TypeOf(reflect.ValueOf(receiver).Elem().Interface()) + if t, ok := registeredReceivers[name]; ok && t != newType { + panic(fmt.Sprintf("duplicate types for %s: %s != %s", name, t, newType)) + } + registeredReceivers[name] = newType +} + +func customReceiverByName(name string) (creceiver CustomReceiver, err error) { + rt, ok := registeredReceivers[name] + if !ok { + return nil, fmt.Errorf("custom receiver name not registered: '%s'", name) + } + v, ok := reflect.New(rt).Interface().(CustomReceiver) + if !ok { + return nil, fmt.Errorf("cannot instantiate receiver with name='%s'", name) + } + return v, nil +} + +// CustomReceiverInitArgs represent arguments passed to the CustomReceiver.Init +// func when custom receiver is being initialized. +type CustomReceiverInitArgs struct { + // XmlCustomAttrs represent '' xml config item attributes that + // start with "data-". Map keys will be the attribute names without the "data-". + // Map values will the those attribute values. + // + // E.g. if you have a '' + // you will get map with 2 key-value pairs: "attr1"->"a1", "attr2"->"a2" + // + // Note that in custom items you can only use allowed attributes, like "name" and + // your custom attributes, starting with "data-". Any other will lead to a + // parsing error. + XmlCustomAttrs map[string]string +} + +// CustomReceiver is the interface that external custom seelog message receivers +// must implement in order to be able to process seelog messages. Those receivers +// are set in the xml config file using the tag. Check receivers reference +// wiki section on that. +// +// Use seelog.RegisterReceiver on the receiver type before using it. +type CustomReceiver interface { + // ReceiveMessage is called when the custom receiver gets seelog message from + // a parent dispatcher. + // + // Message, level and context args represent all data that was included in the seelog + // message at the time it was logged. + // + // The formatting is already applied to the message and depends on the config + // like with any other receiver. + // + // If you would like to inform seelog of an error that happened during the handling of + // the message, return a non-nil error. This way you'll end up seeing your error like + // any other internal seelog error. + ReceiveMessage(message string, level LogLevel, context LogContextInterface) error + + // AfterParse is called immediately after your custom receiver is instantiated by + // the xml config parser. So, if you need to do any startup logic after config parsing, + // like opening file or allocating any resources after the receiver is instantiated, do it here. + // + // If this func returns a non-nil error, then the loading procedure will fail. E.g. + // if you are loading a seelog xml config, the parser would not finish the loading + // procedure and inform about an error like with any other config error. + // + // If your custom logger needs some configuration, you can use custom attributes in + // your config. Check CustomReceiverInitArgs.XmlCustomAttrs comments. + // + // IMPORTANT: This func is NOT called when the LoggerFromCustomReceiver func is used + // to create seelog proxy logger using the custom receiver. This func is only called when + // receiver is instantiated from a config. + AfterParse(initArgs CustomReceiverInitArgs) error + + // Flush is called when the custom receiver gets a 'flush' directive from a + // parent receiver. If custom receiver implements some kind of buffering or + // queing, then the appropriate reaction on a flush message is synchronous + // flushing of all those queues/buffers. If custom receiver doesn't have + // such mechanisms, then flush implementation may be left empty. + Flush() + + // Close is called when the custom receiver gets a 'close' directive from a + // parent receiver. This happens when a top-level seelog dispatcher is sending + // 'close' to all child nodes and it means that current seelog logger is being closed. + // If you need to do any cleanup after your custom receiver is done, you should do + // it here. + Close() error +} + +type customReceiverDispatcher struct { + formatter *formatter + innerReceiver CustomReceiver + customReceiverName string + usedArgs CustomReceiverInitArgs +} + +// NewCustomReceiverDispatcher creates a customReceiverDispatcher which dispatches data to a specific receiver created +// using a tag in the config file. +func NewCustomReceiverDispatcher(formatter *formatter, customReceiverName string, cArgs CustomReceiverInitArgs) (*customReceiverDispatcher, error) { + if formatter == nil { + return nil, errors.New("formatter cannot be nil") + } + if len(customReceiverName) == 0 { + return nil, errors.New("custom receiver name cannot be empty") + } + + creceiver, err := customReceiverByName(customReceiverName) + if err != nil { + return nil, err + } + err = creceiver.AfterParse(cArgs) + if err != nil { + return nil, err + } + disp := &customReceiverDispatcher{formatter, creceiver, customReceiverName, cArgs} + + return disp, nil +} + +// NewCustomReceiverDispatcherByValue is basically the same as NewCustomReceiverDispatcher, but using +// a specific CustomReceiver value instead of instantiating a new one by type. +func NewCustomReceiverDispatcherByValue(formatter *formatter, customReceiver CustomReceiver, name string, cArgs CustomReceiverInitArgs) (*customReceiverDispatcher, error) { + if formatter == nil { + return nil, errors.New("formatter cannot be nil") + } + if customReceiver == nil { + return nil, errors.New("customReceiver cannot be nil") + } + disp := &customReceiverDispatcher{formatter, customReceiver, name, cArgs} + + return disp, nil +} + +// CustomReceiver implementation. Check CustomReceiver comments. +func (disp *customReceiverDispatcher) Dispatch( + message string, + level LogLevel, + context LogContextInterface, + errorFunc func(err error)) { + + defer func() { + if err := recover(); err != nil { + errorFunc(fmt.Errorf("panic in custom receiver '%s'.Dispatch: %s", reflect.TypeOf(disp.innerReceiver), err)) + } + }() + + err := disp.innerReceiver.ReceiveMessage(disp.formatter.Format(message, level, context), level, context) + if err != nil { + errorFunc(err) + } +} + +// CustomReceiver implementation. Check CustomReceiver comments. +func (disp *customReceiverDispatcher) Flush() { + disp.innerReceiver.Flush() +} + +// CustomReceiver implementation. Check CustomReceiver comments. +func (disp *customReceiverDispatcher) Close() error { + disp.innerReceiver.Flush() + + err := disp.innerReceiver.Close() + if err != nil { + return err + } + + return nil +} + +func (disp *customReceiverDispatcher) String() string { + datas := "" + skeys := make([]string, 0, len(disp.usedArgs.XmlCustomAttrs)) + for i := range disp.usedArgs.XmlCustomAttrs { + skeys = append(skeys, i) + } + sort.Strings(skeys) + for _, key := range skeys { + datas += fmt.Sprintf("<%s, %s> ", key, disp.usedArgs.XmlCustomAttrs[key]) + } + + str := fmt.Sprintf("Custom receiver %s [fmt='%s'],[data='%s'],[inner='%s']\n", + disp.customReceiverName, disp.formatter.String(), datas, disp.innerReceiver) + + return str +} diff --git a/lib/seelog/dispatch_customdispatcher_test.go b/lib/seelog/dispatch_customdispatcher_test.go new file mode 100644 index 000000000..23f631a2d --- /dev/null +++ b/lib/seelog/dispatch_customdispatcher_test.go @@ -0,0 +1,177 @@ +// Copyright (c) 2013 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "testing" +) + +type testCustomDispatcherMessageReceiver struct { + customTestReceiver +} + +func TestCustomDispatcher_Message(t *testing.T) { + recName := "TestCustomDispatcher_Message" + RegisterReceiver(recName, &testCustomDispatcherMessageReceiver{}) + + customDispatcher, err := NewCustomReceiverDispatcher(onlyMessageFormatForTest, recName, CustomReceiverInitArgs{ + XmlCustomAttrs: map[string]string{ + "test": "testdata", + }, + }) + if err != nil { + t.Error(err) + return + } + + context, err := currentContext(nil) + if err != nil { + t.Error(err) + return + } + + bytes := []byte("Hello") + customDispatcher.Dispatch(string(bytes), TraceLvl, context, func(err error) {}) + + cout := customDispatcher.innerReceiver.(*testCustomDispatcherMessageReceiver).customTestReceiver.co + if cout.initCalled != true { + t.Error("Init not called") + return + } + if cout.dataPassed != "testdata" { + t.Errorf("wrong data passed: '%s'", cout.dataPassed) + return + } + if cout.messageOutput != string(bytes) { + t.Errorf("wrong message output: '%s'", cout.messageOutput) + return + } + if cout.levelOutput != TraceLvl { + t.Errorf("wrong log level: '%s'", cout.levelOutput) + return + } + if cout.flushed { + t.Error("Flush was not expected") + return + } + if cout.closed { + t.Error("Closing was not expected") + return + } +} + +type testCustomDispatcherFlushReceiver struct { + customTestReceiver +} + +func TestCustomDispatcher_Flush(t *testing.T) { + recName := "TestCustomDispatcher_Flush" + RegisterReceiver(recName, &testCustomDispatcherFlushReceiver{}) + + customDispatcher, err := NewCustomReceiverDispatcher(onlyMessageFormatForTest, recName, CustomReceiverInitArgs{ + XmlCustomAttrs: map[string]string{ + "test": "testdata", + }, + }) + if err != nil { + t.Error(err) + return + } + + customDispatcher.Flush() + + cout := customDispatcher.innerReceiver.(*testCustomDispatcherFlushReceiver).customTestReceiver.co + if cout.initCalled != true { + t.Error("Init not called") + return + } + if cout.dataPassed != "testdata" { + t.Errorf("wrong data passed: '%s'", cout.dataPassed) + return + } + if cout.messageOutput != "" { + t.Errorf("wrong message output: '%s'", cout.messageOutput) + return + } + if cout.levelOutput != TraceLvl { + t.Errorf("wrong log level: '%s'", cout.levelOutput) + return + } + if !cout.flushed { + t.Error("Flush was expected") + return + } + if cout.closed { + t.Error("Closing was not expected") + return + } +} + +type testCustomDispatcherCloseReceiver struct { + customTestReceiver +} + +func TestCustomDispatcher_Close(t *testing.T) { + recName := "TestCustomDispatcher_Close" + RegisterReceiver(recName, &testCustomDispatcherCloseReceiver{}) + + customDispatcher, err := NewCustomReceiverDispatcher(onlyMessageFormatForTest, recName, CustomReceiverInitArgs{ + XmlCustomAttrs: map[string]string{ + "test": "testdata", + }, + }) + if err != nil { + t.Error(err) + return + } + + customDispatcher.Close() + + cout := customDispatcher.innerReceiver.(*testCustomDispatcherCloseReceiver).customTestReceiver.co + if cout.initCalled != true { + t.Error("Init not called") + return + } + if cout.dataPassed != "testdata" { + t.Errorf("wrong data passed: '%s'", cout.dataPassed) + return + } + if cout.messageOutput != "" { + t.Errorf("wrong message output: '%s'", cout.messageOutput) + return + } + if cout.levelOutput != TraceLvl { + t.Errorf("wrong log level: '%s'", cout.levelOutput) + return + } + if !cout.flushed { + t.Error("Flush was expected") + return + } + if !cout.closed { + t.Error("Closing was expected") + return + } +} diff --git a/lib/seelog/dispatch_dispatcher.go b/lib/seelog/dispatch_dispatcher.go new file mode 100644 index 000000000..2bd3b4a4c --- /dev/null +++ b/lib/seelog/dispatch_dispatcher.go @@ -0,0 +1,189 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "fmt" + "io" +) + +// A dispatcherInterface is used to dispatch message to all underlying receivers. +// Dispatch logic depends on given context and log level. Any errors are reported using errorFunc. +// Also, as underlying receivers may have a state, dispatcher has a ShuttingDown method which performs +// an immediate cleanup of all data that is stored in the receivers +type dispatcherInterface interface { + flusherInterface + io.Closer + Dispatch(message string, level LogLevel, context LogContextInterface, errorFunc func(err error)) +} + +type dispatcher struct { + formatter *formatter + writers []*formattedWriter + dispatchers []dispatcherInterface +} + +// Creates a dispatcher which dispatches data to a list of receivers. +// Each receiver should be either a Dispatcher or io.Writer, otherwise an error will be returned +func createDispatcher(formatter *formatter, receivers []interface{}) (*dispatcher, error) { + if formatter == nil { + return nil, errors.New("formatter cannot be nil") + } + if receivers == nil || len(receivers) == 0 { + return nil, errors.New("receivers cannot be nil or empty") + } + + disp := &dispatcher{formatter, make([]*formattedWriter, 0), make([]dispatcherInterface, 0)} + for _, receiver := range receivers { + writer, ok := receiver.(*formattedWriter) + if ok { + disp.writers = append(disp.writers, writer) + continue + } + + ioWriter, ok := receiver.(io.Writer) + if ok { + writer, err := NewFormattedWriter(ioWriter, disp.formatter) + if err != nil { + return nil, err + } + disp.writers = append(disp.writers, writer) + continue + } + + dispInterface, ok := receiver.(dispatcherInterface) + if ok { + disp.dispatchers = append(disp.dispatchers, dispInterface) + continue + } + + return nil, errors.New("method can receive either io.Writer or dispatcherInterface") + } + + return disp, nil +} + +func (disp *dispatcher) Dispatch( + message string, + level LogLevel, + context LogContextInterface, + errorFunc func(err error)) { + + for _, writer := range disp.writers { + err := writer.Write(message, level, context) + if err != nil { + errorFunc(err) + } + } + + for _, dispInterface := range disp.dispatchers { + dispInterface.Dispatch(message, level, context, errorFunc) + } +} + +// Flush goes through all underlying writers which implement flusherInterface interface +// and closes them. Recursively performs the same action for underlying dispatchers +func (disp *dispatcher) Flush() { + for _, disp := range disp.Dispatchers() { + disp.Flush() + } + + for _, formatWriter := range disp.Writers() { + flusher, ok := formatWriter.Writer().(flusherInterface) + if ok { + flusher.Flush() + } + } +} + +// Close goes through all underlying writers which implement io.Closer interface +// and closes them. Recursively performs the same action for underlying dispatchers +// Before closing, writers are flushed to prevent loss of any buffered data, so +// a call to Flush() func before Close() is not necessary +func (disp *dispatcher) Close() error { + for _, disp := range disp.Dispatchers() { + disp.Flush() + err := disp.Close() + if err != nil { + return err + } + } + + for _, formatWriter := range disp.Writers() { + flusher, ok := formatWriter.Writer().(flusherInterface) + if ok { + flusher.Flush() + } + + closer, ok := formatWriter.Writer().(io.Closer) + if ok { + err := closer.Close() + if err != nil { + return err + } + } + } + + return nil +} + +func (disp *dispatcher) Writers() []*formattedWriter { + return disp.writers +} + +func (disp *dispatcher) Dispatchers() []dispatcherInterface { + return disp.dispatchers +} + +func (disp *dispatcher) String() string { + str := "formatter: " + disp.formatter.String() + "\n" + + str += " ->Dispatchers:" + + if len(disp.dispatchers) == 0 { + str += "none\n" + } else { + str += "\n" + + for _, disp := range disp.dispatchers { + str += fmt.Sprintf(" ->%s", disp) + } + } + + str += " ->Writers:" + + if len(disp.writers) == 0 { + str += "none\n" + } else { + str += "\n" + + for _, writer := range disp.writers { + str += fmt.Sprintf(" ->%s\n", writer) + } + } + + return str +} diff --git a/lib/seelog/dispatch_filterdispatcher.go b/lib/seelog/dispatch_filterdispatcher.go new file mode 100644 index 000000000..9de8a7225 --- /dev/null +++ b/lib/seelog/dispatch_filterdispatcher.go @@ -0,0 +1,66 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" +) + +// A filterDispatcher writes the given message to underlying receivers only if message log level +// is in the allowed list. +type filterDispatcher struct { + *dispatcher + allowList map[LogLevel]bool +} + +// NewFilterDispatcher creates a new filterDispatcher using a list of allowed levels. +func NewFilterDispatcher(formatter *formatter, receivers []interface{}, allowList ...LogLevel) (*filterDispatcher, error) { + disp, err := createDispatcher(formatter, receivers) + if err != nil { + return nil, err + } + + allows := make(map[LogLevel]bool) + for _, allowLevel := range allowList { + allows[allowLevel] = true + } + + return &filterDispatcher{disp, allows}, nil +} + +func (filter *filterDispatcher) Dispatch( + message string, + level LogLevel, + context LogContextInterface, + errorFunc func(err error)) { + isAllowed, ok := filter.allowList[level] + if ok && isAllowed { + filter.dispatcher.Dispatch(message, level, context, errorFunc) + } +} + +func (filter *filterDispatcher) String() string { + return fmt.Sprintf("filterDispatcher ->\n%s", filter.dispatcher) +} diff --git a/lib/seelog/dispatch_filterdispatcher_test.go b/lib/seelog/dispatch_filterdispatcher_test.go new file mode 100644 index 000000000..c1894a76b --- /dev/null +++ b/lib/seelog/dispatch_filterdispatcher_test.go @@ -0,0 +1,67 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "testing" +) + +func TestfilterDispatcher_Pass(t *testing.T) { + writer, _ := newBytesVerifier(t) + filter, err := NewFilterDispatcher(onlyMessageFormatForTest, []interface{}{writer}, TraceLvl) + if err != nil { + t.Error(err) + return + } + + context, err := currentContext(nil) + if err != nil { + t.Error(err) + return + } + + bytes := []byte("Hello") + writer.ExpectBytes(bytes) + filter.Dispatch(string(bytes), TraceLvl, context, func(err error) {}) + writer.MustNotExpect() +} + +func TestfilterDispatcher_Deny(t *testing.T) { + writer, _ := newBytesVerifier(t) + filter, err := NewFilterDispatcher(DefaultFormatter, []interface{}{writer}) + if err != nil { + t.Error(err) + return + } + + context, err := currentContext(nil) + if err != nil { + t.Error(err) + return + } + + bytes := []byte("Hello") + filter.Dispatch(string(bytes), TraceLvl, context, func(err error) {}) +} diff --git a/lib/seelog/dispatch_splitdispatcher.go b/lib/seelog/dispatch_splitdispatcher.go new file mode 100644 index 000000000..1d0fe7eac --- /dev/null +++ b/lib/seelog/dispatch_splitdispatcher.go @@ -0,0 +1,47 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" +) + +// A splitDispatcher just writes the given message to underlying receivers. (Splits the message stream.) +type splitDispatcher struct { + *dispatcher +} + +func NewSplitDispatcher(formatter *formatter, receivers []interface{}) (*splitDispatcher, error) { + disp, err := createDispatcher(formatter, receivers) + if err != nil { + return nil, err + } + + return &splitDispatcher{disp}, nil +} + +func (splitter *splitDispatcher) String() string { + return fmt.Sprintf("splitDispatcher ->\n%s", splitter.dispatcher.String()) +} diff --git a/lib/seelog/dispatch_splitdispatcher_test.go b/lib/seelog/dispatch_splitdispatcher_test.go new file mode 100644 index 000000000..fc4651c2c --- /dev/null +++ b/lib/seelog/dispatch_splitdispatcher_test.go @@ -0,0 +1,64 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" + "testing" +) + +var onlyMessageFormatForTest *formatter + +func init() { + var err error + onlyMessageFormatForTest, err = NewFormatter("%Msg") + if err != nil { + fmt.Println("Can not create only message format: " + err.Error()) + } +} + +func TestsplitDispatcher(t *testing.T) { + writer1, _ := newBytesVerifier(t) + writer2, _ := newBytesVerifier(t) + spliter, err := NewSplitDispatcher(onlyMessageFormatForTest, []interface{}{writer1, writer2}) + if err != nil { + t.Error(err) + return + } + + context, err := currentContext(nil) + if err != nil { + t.Error(err) + return + } + + bytes := []byte("Hello") + + writer1.ExpectBytes(bytes) + writer2.ExpectBytes(bytes) + spliter.Dispatch(string(bytes), TraceLvl, context, func(err error) {}) + writer1.MustNotExpect() + writer2.MustNotExpect() +} diff --git a/lib/seelog/doc.go b/lib/seelog/doc.go new file mode 100644 index 000000000..b3e5399dc --- /dev/null +++ b/lib/seelog/doc.go @@ -0,0 +1,190 @@ +// Copyright (c) 2014 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +/* +Package seelog implements logging functionality with flexible dispatching, filtering, and formatting. + +# Creation + +To create a logger, use one of the following constructors: + + func LoggerFromConfigAsBytes + func LoggerFromConfigAsFile + func LoggerFromConfigAsString + func LoggerFromWriterWithMinLevel + func LoggerFromWriterWithMinLevelAndFormat + func LoggerFromCustomReceiver (check https://github.com/cihub/seelog/wiki/Custom-receivers) + +Example: + + import log "github.com/cihub/seelog" + + func main() { + logger, err := log.LoggerFromConfigAsFile("seelog.xml") + if err != nil { + panic(err) + } + defer logger.Flush() + ... use logger ... + } + +The "defer" line is important because if you are using asynchronous logger behavior, without this line you may end up losing some +messages when you close your application because they are processed in another non-blocking goroutine. To avoid that you +explicitly defer flushing all messages before closing. + +# Usage + +Logger created using one of the LoggerFrom* funcs can be used directly by calling one of the main log funcs. +Example: + + import log "github.com/cihub/seelog" + + func main() { + logger, err := log.LoggerFromConfigAsFile("seelog.xml") + if err != nil { + panic(err) + } + defer logger.Flush() + logger.Trace("test") + logger.Debugf("var = %s", "abc") + } + +Having loggers as variables is convenient if you are writing your own package with internal logging or if you have +several loggers with different options. +But for most standalone apps it is more convenient to use package level funcs and vars. There is a package level +var 'Current' made for it. You can replace it with another logger using 'ReplaceLogger' and then use package level funcs: + + import log "github.com/cihub/seelog" + + func main() { + logger, err := log.LoggerFromConfigAsFile("seelog.xml") + if err != nil { + panic(err) + } + log.ReplaceLogger(logger) + defer log.Flush() + log.Trace("test") + log.Debugf("var = %s", "abc") + } + +Last lines + + log.Trace("test") + log.Debugf("var = %s", "abc") + +do the same as + + log.Current.Trace("test") + log.Current.Debugf("var = %s", "abc") + +In this example the 'Current' logger was replaced using a 'ReplaceLogger' call and became equal to 'logger' variable created from config. +This way you are able to use package level funcs instead of passing the logger variable. + +# Configuration + +Main seelog point is to configure logger via config files and not the code. +The configuration is read by LoggerFrom* funcs. These funcs read xml configuration from different sources and try +to create a logger using it. + +All the configuration features are covered in detail in the official wiki: https://github.com/cihub/seelog/wiki. +There are many sections covering different aspects of seelog, but the most important for understanding configs are: + + https://github.com/cihub/seelog/wiki/Constraints-and-exceptions + https://github.com/cihub/seelog/wiki/Dispatchers-and-receivers + https://github.com/cihub/seelog/wiki/Formatting + https://github.com/cihub/seelog/wiki/Logger-types + +After you understand these concepts, check the 'Reference' section on the main wiki page to get the up-to-date +list of dispatchers, receivers, formats, and logger types. + +Here is an example config with all these features: + + + + + + + + + + + + + + + + + + + + + + + +This config represents a logger with adaptive timeout between log messages (check logger types reference) which +logs to console, all.log, and errors.log depending on the log level. Its output formats also depend on log level. This logger will only +use log level 'debug' and higher (minlevel is set) for all files with names that don't start with 'test'. For files starting with 'test' +this logger prohibits all levels below 'error'. + +# Configuration using code + +Although configuration using code is not recommended, it is sometimes needed and it is possible to do with seelog. Basically, what +you need to do to get started is to create constraints, exceptions and a dispatcher tree (same as with config). Most of the New* +functions in this package are used to provide such capabilities. + +Here is an example of configuration in code, that demonstrates an async loop logger that logs to a simple split dispatcher with +a console receiver using a specified format and is filtered using a top-level min-max constraints and one expection for +the 'main.go' file. So, this is basically a demonstration of configuration of most of the features: + + package main + + import log "github.com/cihub/seelog" + + func main() { + defer log.Flush() + log.Info("Hello from Seelog!") + + consoleWriter, _ := log.NewConsoleWriter() + formatter, _ := log.NewFormatter("%Level %Msg %File%n") + root, _ := log.NewSplitDispatcher(formatter, []interface{}{consoleWriter}) + constraints, _ := log.NewMinMaxConstraints(log.TraceLvl, log.CriticalLvl) + specificConstraints, _ := log.NewListConstraints([]log.LogLevel{log.InfoLvl, log.ErrorLvl}) + ex, _ := log.NewLogLevelException("*", "*main.go", specificConstraints) + exceptions := []*log.LogLevelException{ex} + + logger := log.NewAsyncLoopLogger(log.NewLoggerConfig(constraints, exceptions, root)) + log.ReplaceLogger(logger) + + log.Trace("This should not be seen") + log.Debug("This should not be seen") + log.Info("Test") + log.Error("Test2") + } + +# Examples + +To learn seelog features faster you should check the examples package: https://github.com/cihub/seelog-examples +It contains many example configs and usecases. +*/ +package seelog diff --git a/lib/seelog/format.go b/lib/seelog/format.go new file mode 100644 index 000000000..6e82d312a --- /dev/null +++ b/lib/seelog/format.go @@ -0,0 +1,461 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// FormatterSymbol is a special symbol used in config files to mark special format aliases. +const ( + FormatterSymbol = '%' +) + +const ( + formatterParameterStart = '(' + formatterParameterEnd = ')' +) + +// Time and date formats used for %Date and %Time aliases. +const ( + DateDefaultFormat = "2006-01-02" + TimeFormat = "15:04:05" +) + +var DefaultMsgFormat = "[%Date(01-02) %Time] [%LEV] [%File:%Line] %Msg%n" + +var ( + DefaultFormatter *formatter + msgonlyformatter *formatter +) + +func init() { + var err error + if DefaultFormatter, err = NewFormatter(DefaultMsgFormat); err != nil { + reportInternalError(fmt.Errorf("error during creating DefaultFormatter: %s", err)) + } + if msgonlyformatter, err = NewFormatter("%Msg"); err != nil { + reportInternalError(fmt.Errorf("error during creating msgonlyformatter: %s", err)) + } +} + +// FormatterFunc represents one formatter object that starts with '%' sign in the 'format' attribute +// of the 'format' config item. These special symbols are replaced with context values or special +// strings when message is written to byte receiver. +// +// Check https://github.com/cihub/seelog/wiki/Formatting for details. +// Full list (with descriptions) of formatters: https://github.com/cihub/seelog/wiki/Format-reference +// +// FormatterFunc takes raw log message, level, log context and returns a string, number (of any type) or any object +// that can be evaluated as string. +type FormatterFunc func(message string, level LogLevel, context LogContextInterface) interface{} + +// FormatterFuncCreator is a factory of FormatterFunc objects. It is used to generate parameterized +// formatters (such as %Date or %EscM) and custom user formatters. +type FormatterFuncCreator func(param string) FormatterFunc + +var formatterFuncs = map[string]FormatterFunc{ + "Level": formatterLevel, + "Lev": formatterLev, + "LEVEL": formatterLEVEL, + "LEV": formatterLEV, + "l": formatterl, + "Msg": formatterMsg, + "FullPath": formatterFullPath, + "File": formatterFile, + "RelFile": formatterRelFile, + "Func": FormatterFunction, + "FuncShort": FormatterFunctionShort, + "Line": formatterLine, + "Time": formatterTime, + "UTCTime": formatterUTCTime, + "Ns": formatterNs, + "UTCNs": formatterUTCNs, + "n": formattern, + "t": formattert, +} + +var formatterFuncsParameterized = map[string]FormatterFuncCreator{ + "Date": createDateTimeFormatterFunc, + "UTCDate": createUTCDateTimeFormatterFunc, + "EscM": createANSIEscapeFunc, +} + +func errorAliasReserved(name string) error { + return fmt.Errorf("cannot use '%s' as custom formatter name. Name is reserved", name) +} + +// RegisterCustomFormatter registers a new custom formatter factory with a given name. If returned error is nil, +// then this name (prepended by '%' symbol) can be used in 'format' attributes in configuration and +// it will be treated like the standard parameterized formatter identifiers. +// +// RegisterCustomFormatter needs to be called before creating a logger for it to take effect. The general recommendation +// is to call it once in 'init' func of your application or any initializer func. +// +// For usage examples, check https://github.com/cihub/seelog/wiki/Custom-formatters. +// +// Name must only consist of letters (unicode.IsLetter). +// +// Name must not be one of the already registered standard formatter names +// (https://github.com/cihub/seelog/wiki/Format-reference) and previously registered +// custom format names. To avoid any potential name conflicts (in future releases), it is recommended +// to start your custom formatter name with a namespace (e.g. 'MyCompanySomething') or a 'Custom' keyword. +func RegisterCustomFormatter(name string, creator FormatterFuncCreator) error { + if _, ok := formatterFuncs[name]; ok { + return errorAliasReserved(name) + } + if _, ok := formatterFuncsParameterized[name]; ok { + return errorAliasReserved(name) + } + formatterFuncsParameterized[name] = creator + return nil +} + +// formatter is used to write messages in a specific format, inserting such additional data +// as log level, date/time, etc. +type formatter struct { + fmtStringOriginal string + fmtString string + formatterFuncs []FormatterFunc +} + +// NewFormatter creates a new formatter using a format string +func NewFormatter(formatString string) (*formatter, error) { + fmtr := new(formatter) + fmtr.fmtStringOriginal = formatString + if err := buildFormatterFuncs(fmtr); err != nil { + return nil, err + } + return fmtr, nil +} + +func buildFormatterFuncs(formatter *formatter) error { + var ( + fsbuf = new(bytes.Buffer) + fsolm1 = len(formatter.fmtStringOriginal) - 1 + ) + for i := 0; i <= fsolm1; i++ { + if char := formatter.fmtStringOriginal[i]; char != FormatterSymbol { + fsbuf.WriteByte(char) + continue + } + // Check if the index is at the end of the string. + if i == fsolm1 { + return fmt.Errorf("format error: %c cannot be last symbol", FormatterSymbol) + } + // Check if the formatter symbol is doubled and skip it as nonmatching. + if formatter.fmtStringOriginal[i+1] == FormatterSymbol { + fsbuf.WriteRune(FormatterSymbol) + i++ + continue + } + function, ni, err := formatter.extractFormatterFunc(i + 1) + if err != nil { + return err + } + // Append formatting string "%v". + fsbuf.Write([]byte{37, 118}) + i = ni + formatter.formatterFuncs = append(formatter.formatterFuncs, function) + } + formatter.fmtString = fsbuf.String() + return nil +} + +func (formatter *formatter) extractFormatterFunc(index int) (FormatterFunc, int, error) { + letterSequence := formatter.extractLetterSequence(index) + if len(letterSequence) == 0 { + return nil, 0, fmt.Errorf("format error: lack of formatter after %c at %d", FormatterSymbol, index) + } + + function, formatterLength, ok := formatter.findFormatterFunc(letterSequence) + if ok { + return function, index + formatterLength - 1, nil + } + + function, formatterLength, ok, err := formatter.findFormatterFuncParametrized(letterSequence, index) + if err != nil { + return nil, 0, err + } + if ok { + return function, index + formatterLength - 1, nil + } + + return nil, 0, errors.New("format error: unrecognized formatter at " + strconv.Itoa(index) + ": " + letterSequence) +} + +func (formatter *formatter) extractLetterSequence(index int) string { + letters := "" + + bytesToParse := []byte(formatter.fmtStringOriginal[index:]) + runeCount := utf8.RuneCount(bytesToParse) + for i := 0; i < runeCount; i++ { + rune, runeSize := utf8.DecodeRune(bytesToParse) + bytesToParse = bytesToParse[runeSize:] + + if unicode.IsLetter(rune) { + letters += string(rune) + } else { + break + } + } + return letters +} + +func (formatter *formatter) findFormatterFunc(letters string) (FormatterFunc, int, bool) { + currentVerb := letters + for i := 0; i < len(letters); i++ { + function, ok := formatterFuncs[currentVerb] + if ok { + return function, len(currentVerb), ok + } + currentVerb = currentVerb[:len(currentVerb)-1] + } + + return nil, 0, false +} + +func (formatter *formatter) findFormatterFuncParametrized(letters string, lettersStartIndex int) (FormatterFunc, int, bool, error) { + currentVerb := letters + for i := 0; i < len(letters); i++ { + functionCreator, ok := formatterFuncsParameterized[currentVerb] + if ok { + parameter := "" + parameterLen := 0 + isVerbEqualsLetters := i == 0 // if not, then letter goes after formatter, and formatter is parameterless + if isVerbEqualsLetters { + userParameter := "" + var err error + userParameter, parameterLen, ok, err = formatter.findparameter(lettersStartIndex + len(currentVerb)) + if ok { + parameter = userParameter + } else if err != nil { + return nil, 0, false, err + } + } + + return functionCreator(parameter), len(currentVerb) + parameterLen, true, nil + } + + currentVerb = currentVerb[:len(currentVerb)-1] + } + + return nil, 0, false, nil +} + +func (formatter *formatter) findparameter(startIndex int) (string, int, bool, error) { + if len(formatter.fmtStringOriginal) == startIndex || formatter.fmtStringOriginal[startIndex] != formatterParameterStart { + return "", 0, false, nil + } + + endIndex := strings.Index(formatter.fmtStringOriginal[startIndex:], string(formatterParameterEnd)) + if endIndex == -1 { + return "", 0, false, fmt.Errorf("Unmatched parenthesis or invalid parameter at %d: %s", + startIndex, formatter.fmtStringOriginal[startIndex:]) + } + endIndex += startIndex + + length := endIndex - startIndex + 1 + + return formatter.fmtStringOriginal[startIndex+1 : endIndex], length, true, nil +} + +// Format processes a message with special formatters, log level, and context. Returns formatted string +// with all formatter identifiers changed to appropriate values. +func (formatter *formatter) Format(message string, level LogLevel, context LogContextInterface) string { + if len(formatter.formatterFuncs) == 0 { + return formatter.fmtString + } + + params := make([]interface{}, len(formatter.formatterFuncs)) + for i, function := range formatter.formatterFuncs { + params[i] = function(message, level, context) + } + + return fmt.Sprintf(formatter.fmtString, params...) +} + +func (formatter *formatter) String() string { + return formatter.fmtStringOriginal +} + +//===================================================== + +const ( + wrongLogLevel = "WRONG_LOGLEVEL" + wrongEscapeCode = "WRONG_ESCAPE" +) + +var levelToString = map[LogLevel]string{ + TraceLvl: "Trace", + DebugLvl: "Debug", + InfoLvl: "Info", + WarnLvl: "Warn", + ErrorLvl: "Error", + CriticalLvl: "Critical", + Off: "Off", +} + +var levelToShortString = map[LogLevel]string{ + TraceLvl: "Trc", + DebugLvl: "Dbg", + InfoLvl: "Inf", + WarnLvl: "Wrn", + ErrorLvl: "Err", + CriticalLvl: "Crt", + Off: "Off", +} + +var levelToShortestString = map[LogLevel]string{ + TraceLvl: "t", + DebugLvl: "d", + InfoLvl: "i", + WarnLvl: "w", + ErrorLvl: "e", + CriticalLvl: "c", + Off: "o", +} + +func formatterLevel(message string, level LogLevel, context LogContextInterface) interface{} { + levelStr, ok := levelToString[level] + if !ok { + return wrongLogLevel + } + return levelStr +} + +func formatterLev(message string, level LogLevel, context LogContextInterface) interface{} { + levelStr, ok := levelToShortString[level] + if !ok { + return wrongLogLevel + } + return levelStr +} + +func formatterLEVEL(message string, level LogLevel, context LogContextInterface) interface{} { + return strings.ToTitle(formatterLevel(message, level, context).(string)) +} + +func formatterLEV(message string, level LogLevel, context LogContextInterface) interface{} { + return strings.ToTitle(formatterLev(message, level, context).(string)) +} + +func formatterl(message string, level LogLevel, context LogContextInterface) interface{} { + levelStr, ok := levelToShortestString[level] + if !ok { + return wrongLogLevel + } + return levelStr +} + +func formatterMsg(message string, level LogLevel, context LogContextInterface) interface{} { + return message +} + +func formatterFullPath(message string, level LogLevel, context LogContextInterface) interface{} { + return context.FullPath() +} + +func formatterFile(message string, level LogLevel, context LogContextInterface) interface{} { + return context.FileName() +} + +func formatterRelFile(message string, level LogLevel, context LogContextInterface) interface{} { + return context.ShortPath() +} + +func FormatterFunction(message string, level LogLevel, context LogContextInterface) interface{} { + return context.Func() +} + +func FormatterFunctionShort(message string, level LogLevel, context LogContextInterface) interface{} { + f := context.Func() + spl := strings.Split(f, ".") + return spl[len(spl)-1] +} + +func formatterLine(message string, level LogLevel, context LogContextInterface) interface{} { + return context.Line() +} + +func formatterTime(message string, level LogLevel, context LogContextInterface) interface{} { + return context.CallTime().Format(TimeFormat) +} + +func formatterUTCTime(message string, level LogLevel, context LogContextInterface) interface{} { + return context.CallTime().UTC().Format(TimeFormat) +} + +func formatterNs(message string, level LogLevel, context LogContextInterface) interface{} { + return context.CallTime().UnixNano() +} + +func formatterUTCNs(message string, level LogLevel, context LogContextInterface) interface{} { + return context.CallTime().UTC().UnixNano() +} + +func formattern(message string, level LogLevel, context LogContextInterface) interface{} { + return "\n" +} + +func formattert(message string, level LogLevel, context LogContextInterface) interface{} { + return "\t" +} + +func createDateTimeFormatterFunc(dateTimeFormat string) FormatterFunc { + format := dateTimeFormat + if format == "" { + format = DateDefaultFormat + } + return func(message string, level LogLevel, context LogContextInterface) interface{} { + return context.CallTime().Format(format) + } +} + +func createUTCDateTimeFormatterFunc(dateTimeFormat string) FormatterFunc { + format := dateTimeFormat + if format == "" { + format = DateDefaultFormat + } + return func(message string, level LogLevel, context LogContextInterface) interface{} { + return context.CallTime().UTC().Format(format) + } +} + +func createANSIEscapeFunc(escapeCodeString string) FormatterFunc { + return func(message string, level LogLevel, context LogContextInterface) interface{} { + if len(escapeCodeString) == 0 { + return wrongEscapeCode + } + + return fmt.Sprintf("%c[%sm", 0x1B, escapeCodeString) + } +} diff --git a/lib/seelog/format_test.go b/lib/seelog/format_test.go new file mode 100644 index 000000000..dd61bdfd5 --- /dev/null +++ b/lib/seelog/format_test.go @@ -0,0 +1,236 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" + "strings" + "testing" + "time" +) + +const ( + TestFuncName = "TestFormats" +) + +type formatTest struct { + formatString string + input string + inputLogLevel LogLevel + expectedOutput string + errorExpected bool +} + +var formatTests = []formatTest{ + {"test", "abcdef", TraceLvl, "test", false}, + {"", "abcdef", TraceLvl, "", false}, + {"%Level", "", TraceLvl, "Trace", false}, + {"%Level", "", DebugLvl, "Debug", false}, + {"%Level", "", InfoLvl, "Info", false}, + {"%Level", "", WarnLvl, "Warn", false}, + {"%Level", "", ErrorLvl, "Error", false}, + {"%Level", "", CriticalLvl, "Critical", false}, + {"[%Level]", "", TraceLvl, "[Trace]", false}, + {"[%Level]", "abc", DebugLvl, "[Debug]", false}, + {"%LevelLevel", "", InfoLvl, "InfoLevel", false}, + {"[%Level][%Level]", "", WarnLvl, "[Warn][Warn]", false}, + {"[%Level]X[%Level]", "", ErrorLvl, "[Error]X[Error]", false}, + {"%Levelll", "", CriticalLvl, "Criticalll", false}, + {"%Lvl", "", TraceLvl, "", true}, + {"%%Level", "", DebugLvl, "%Level", false}, + {"%Level%", "", InfoLvl, "", true}, + {"%sevel", "", WarnLvl, "", true}, + {"Level", "", ErrorLvl, "Level", false}, + {"%LevelLevel", "", CriticalLvl, "CriticalLevel", false}, + {"%Lev", "", TraceLvl, "Trc", false}, + {"%Lev", "", DebugLvl, "Dbg", false}, + {"%Lev", "", InfoLvl, "Inf", false}, + {"%Lev", "", WarnLvl, "Wrn", false}, + {"%Lev", "", ErrorLvl, "Err", false}, + {"%Lev", "", CriticalLvl, "Crt", false}, + {"[%Lev]", "", TraceLvl, "[Trc]", false}, + {"[%Lev]", "abc", DebugLvl, "[Dbg]", false}, + {"%LevLevel", "", InfoLvl, "InfLevel", false}, + {"[%Level][%Lev]", "", WarnLvl, "[Warn][Wrn]", false}, + {"[%Lev]X[%Lev]", "", ErrorLvl, "[Err]X[Err]", false}, + {"%Levll", "", CriticalLvl, "Crtll", false}, + {"%LEVEL", "", TraceLvl, "TRACE", false}, + {"%LEVEL", "", DebugLvl, "DEBUG", false}, + {"%LEVEL", "", InfoLvl, "INFO", false}, + {"%LEVEL", "", WarnLvl, "WARN", false}, + {"%LEVEL", "", ErrorLvl, "ERROR", false}, + {"%LEVEL", "", CriticalLvl, "CRITICAL", false}, + {"[%LEVEL]", "", TraceLvl, "[TRACE]", false}, + {"[%LEVEL]", "abc", DebugLvl, "[DEBUG]", false}, + {"%LEVELLEVEL", "", InfoLvl, "INFOLEVEL", false}, + {"[%LEVEL][%LEVEL]", "", WarnLvl, "[WARN][WARN]", false}, + {"[%LEVEL]X[%Level]", "", ErrorLvl, "[ERROR]X[Error]", false}, + {"%LEVELLL", "", CriticalLvl, "CRITICALLL", false}, + {"%LEV", "", TraceLvl, "TRC", false}, + {"%LEV", "", DebugLvl, "DBG", false}, + {"%LEV", "", InfoLvl, "INF", false}, + {"%LEV", "", WarnLvl, "WRN", false}, + {"%LEV", "", ErrorLvl, "ERR", false}, + {"%LEV", "", CriticalLvl, "CRT", false}, + {"[%LEV]", "", TraceLvl, "[TRC]", false}, + {"[%LEV]", "abc", DebugLvl, "[DBG]", false}, + {"%LEVLEVEL", "", InfoLvl, "INFLEVEL", false}, + {"[%LEVEL][%LEV]", "", WarnLvl, "[WARN][WRN]", false}, + {"[%LEV]X[%LEV]", "", ErrorLvl, "[ERR]X[ERR]", false}, + {"%LEVLL", "", CriticalLvl, "CRTLL", false}, + {"%l", "", TraceLvl, "t", false}, + {"%l", "", DebugLvl, "d", false}, + {"%l", "", InfoLvl, "i", false}, + {"%l", "", WarnLvl, "w", false}, + {"%l", "", ErrorLvl, "e", false}, + {"%l", "", CriticalLvl, "c", false}, + {"[%l]", "", TraceLvl, "[t]", false}, + {"[%l]", "abc", DebugLvl, "[d]", false}, + {"%Level%Msg", "", TraceLvl, "Trace", false}, + {"%Level%Msg", "A", DebugLvl, "DebugA", false}, + {"%Level%Msg", "", InfoLvl, "Info", false}, + {"%Level%Msg", "test", WarnLvl, "Warntest", false}, + {"%Level%Msg", " ", ErrorLvl, "Error ", false}, + {"%Level%Msg", "", CriticalLvl, "Critical", false}, + {"[%Level]", "", TraceLvl, "[Trace]", false}, + {"[%Level]", "abc", DebugLvl, "[Debug]", false}, + {"%Level%MsgLevel", "A", InfoLvl, "InfoALevel", false}, + {"[%Level]%Msg[%Level]", "test", WarnLvl, "[Warn]test[Warn]", false}, + {"[%Level]%MsgX[%Level]", "test", ErrorLvl, "[Error]testX[Error]", false}, + {"%Levell%Msgl", "Test", CriticalLvl, "CriticallTestl", false}, + {"%Lev%Msg%LEVEL%LEV%l%Msg", "Test", InfoLvl, "InfTestINFOINFiTest", false}, + {"%n", "", CriticalLvl, "\n", false}, + {"%t", "", CriticalLvl, "\t", false}, +} + +func TestFormats(t *testing.T) { + + context, conErr := currentContext(nil) + if conErr != nil { + t.Fatal("Cannot get current context:" + conErr.Error()) + return + } + + for _, test := range formatTests { + + form, err := NewFormatter(test.formatString) + + if (err != nil) != test.errorExpected { + t.Errorf("input: %s \nInput LL: %s\n* Expected error:%t Got error: %t\n", + test.input, test.inputLogLevel, test.errorExpected, (err != nil)) + if err != nil { + t.Logf("%s\n", err.Error()) + } + continue + } else if err != nil { + continue + } + + msg := form.Format(test.input, test.inputLogLevel, context) + + if err == nil && msg != test.expectedOutput { + t.Errorf("format: %s \nInput: %s \nInput LL: %s\n* Expected: %s \n* Got: %s\n", + test.formatString, test.input, test.inputLogLevel, test.expectedOutput, msg) + } + } +} + +func TestDateFormat(t *testing.T) { + _, err := NewFormatter("%Date") + if err != nil { + t.Error("Unexpected error: " + err.Error()) + } +} + +func TestDateParameterizedFormat(t *testing.T) { + testFormat := "Mon Jan 02 2006 15:04:05" + preciseForamt := "Mon Jan 02 2006 15:04:05.000" + + context, conErr := currentContext(nil) + if conErr != nil { + t.Fatal("Cannot get current context:" + conErr.Error()) + return + } + + form, err := NewFormatter("%Date(" + preciseForamt + ")") + if err != nil { + t.Error("Unexpected error: " + err.Error()) + } + + dateBefore := time.Now().Format(testFormat) + msg := form.Format("", TraceLvl, context) + dateAfter := time.Now().Format(testFormat) + + if !strings.HasPrefix(msg, dateBefore) && !strings.HasPrefix(msg, dateAfter) { + t.Errorf("incorrect message: %v. Expected %v or %v", msg, dateBefore, dateAfter) + } + + _, err = NewFormatter("%Date(" + preciseForamt) + if err == nil { + t.Error("Expected error for invalid format") + } +} + +func createTestFormatter(format string) FormatterFunc { + return func(message string, level LogLevel, context LogContextInterface) interface{} { + return "TEST " + context.Func() + " TEST" + } +} + +func TestCustomFormatterRegistration(t *testing.T) { + err := RegisterCustomFormatter("Level", createTestFormatter) + if err == nil { + t.Errorf("expected an error when trying to register a custom formatter with a reserved alias") + } + err = RegisterCustomFormatter("EscM", createTestFormatter) + if err == nil { + t.Errorf("expected an error when trying to register a custom formatter with a reserved parameterized alias") + } + err = RegisterCustomFormatter("TEST", createTestFormatter) + if err != nil { + t.Fatalf("Registering custom formatter: unexpected error: %s", err) + } + err = RegisterCustomFormatter("TEST", createTestFormatter) + if err == nil { + t.Errorf("expected an error when trying to register a custom formatter with duplicate name") + } + + context, conErr := currentContext(nil) + if conErr != nil { + t.Fatal("Cannot get current context:" + conErr.Error()) + return + } + + form, err := NewFormatter("%Msg %TEST 123") + if err != nil { + t.Fatalf("%s\n", err.Error()) + } + + expected := fmt.Sprintf("test TEST %sTestCustomFormatterRegistration TEST 123", commonPrefix) + msg := form.Format("test", DebugLvl, context) + if msg != expected { + t.Fatalf("Custom formatter: invalid output. Expected: '%s'. Got: '%s'", expected, msg) + } +} diff --git a/lib/seelog/go.mod b/lib/seelog/go.mod new file mode 100644 index 000000000..dc5d2bfac --- /dev/null +++ b/lib/seelog/go.mod @@ -0,0 +1,3 @@ +module github.com/cihub/seelog + +go 1.25.0 diff --git a/lib/seelog/internals_baseerror.go b/lib/seelog/internals_baseerror.go new file mode 100644 index 000000000..c0b271d7d --- /dev/null +++ b/lib/seelog/internals_baseerror.go @@ -0,0 +1,10 @@ +package seelog + +// Base struct for custom errors. +type baseError struct { + message string +} + +func (be baseError) Error() string { + return be.message +} diff --git a/lib/seelog/internals_byteverifiers_test.go b/lib/seelog/internals_byteverifiers_test.go new file mode 100644 index 000000000..0ab6ebc68 --- /dev/null +++ b/lib/seelog/internals_byteverifiers_test.go @@ -0,0 +1,118 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "strconv" + "testing" +) + +// bytesVerifier is a byte receiver which is used for correct input testing. +// It allows to compare expected result and actual result in context of received bytes. +type bytesVerifier struct { + expectedBytes []byte // bytes that are expected to be written in next Write call + waitingForInput bool // true if verifier is waiting for a Write call + writtenData []byte // real bytes that actually were received during the last Write call + testEnv *testing.T +} + +func newBytesVerifier(t *testing.T) (*bytesVerifier, error) { + if t == nil { + return nil, errors.New("testing environment param is nil") + } + + verifier := new(bytesVerifier) + verifier.testEnv = t + + return verifier, nil +} + +// Write is used to check whether verifier was waiting for input and whether bytes are the same as expectedBytes. +// After Write call, waitingForInput is set to false. +func (verifier *bytesVerifier) Write(bytes []byte) (n int, err error) { + if !verifier.waitingForInput { + verifier.testEnv.Errorf("unexpected input: %v", string(bytes)) + return + } + + verifier.waitingForInput = false + verifier.writtenData = bytes + + if verifier.expectedBytes != nil { + if bytes == nil { + verifier.testEnv.Errorf("incoming 'bytes' is nil") + } else { + if len(bytes) != len(verifier.expectedBytes) { + verifier.testEnv.Errorf("'Bytes' has unexpected len. Expected: %d. Got: %d. . Expected string: %q. Got: %q", + len(verifier.expectedBytes), len(bytes), string(verifier.expectedBytes), string(bytes)) + } else { + for i := 0; i < len(bytes); i++ { + if verifier.expectedBytes[i] != bytes[i] { + verifier.testEnv.Errorf("incorrect data on position %d. Expected: %d. Got: %d. Expected string: %q. Got: %q", + i, verifier.expectedBytes[i], bytes[i], string(verifier.expectedBytes), string(bytes)) + break + } + } + } + } + } + + return len(bytes), nil +} + +func (verifier *bytesVerifier) ExpectBytes(bytes []byte) { + verifier.waitingForInput = true + verifier.expectedBytes = bytes +} + +func (verifier *bytesVerifier) MustNotExpect() { + if verifier.waitingForInput { + errorText := "Unexpected input: " + + if verifier.expectedBytes != nil { + errorText += "len = " + strconv.Itoa(len(verifier.expectedBytes)) + errorText += ". text = " + string(verifier.expectedBytes) + } + + verifier.testEnv.Errorf(errorText) + } +} + +func (verifier *bytesVerifier) Close() error { + return nil +} + +// nullWriter implements io.Writer inteface and does nothing, always returning a successful write result +type nullWriter struct { +} + +func (writer *nullWriter) Write(bytes []byte) (n int, err error) { + return len(bytes), nil +} + +func (writer *nullWriter) Close() error { + return nil +} diff --git a/lib/seelog/internals_fsutils.go b/lib/seelog/internals_fsutils.go new file mode 100644 index 000000000..5baa6ba61 --- /dev/null +++ b/lib/seelog/internals_fsutils.go @@ -0,0 +1,403 @@ +package seelog + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "sync" +) + +// File and directory permitions. +const ( + defaultFilePermissions = 0666 + defaultDirectoryPermissions = 0767 +) + +const ( + // Max number of directories can be read asynchronously. + maxDirNumberReadAsync = 1000 +) + +type cannotOpenFileError struct { + baseError +} + +func newCannotOpenFileError(fname string) *cannotOpenFileError { + return &cannotOpenFileError{baseError{message: "Cannot open file: " + fname}} +} + +type notDirectoryError struct { + baseError +} + +func newNotDirectoryError(dname string) *notDirectoryError { + return ¬DirectoryError{baseError{message: dname + " is not directory"}} +} + +// fileFilter is a filtering criteria function for '*os.File'. +// Must return 'false' to set aside the given file. +type fileFilter func(os.FileInfo, *os.File) bool + +// filePathFilter is a filtering creteria function for file path. +// Must return 'false' to set aside the given file. +type filePathFilter func(filePath string) bool + +// GetSubdirNames returns a list of directories found in +// the given one with dirPath. +func getSubdirNames(dirPath string) ([]string, error) { + fi, err := os.Stat(dirPath) + if err != nil { + return nil, err + } + if !fi.IsDir() { + return nil, newNotDirectoryError(dirPath) + } + dd, err := os.Open(dirPath) + // Cannot open file. + if err != nil { + if dd != nil { + dd.Close() + } + return nil, err + } + defer dd.Close() + // TODO: Improve performance by buffering reading. + allEntities, err := dd.Readdir(-1) + if err != nil { + return nil, err + } + subDirs := []string{} + for _, entity := range allEntities { + if entity.IsDir() { + subDirs = append(subDirs, entity.Name()) + } + } + return subDirs, nil +} + +// getSubdirAbsPaths recursively visit all the subdirectories +// starting from the given directory and returns absolute paths for them. +func getAllSubdirAbsPaths(dirPath string) (res []string, err error) { + dps, err := getSubdirAbsPaths(dirPath) + if err != nil { + res = []string{} + return + } + res = append(res, dps...) + for _, dp := range dps { + sdps, err := getAllSubdirAbsPaths(dp) + if err != nil { + return []string{}, err + } + res = append(res, sdps...) + } + return +} + +// getSubdirAbsPaths supplies absolute paths for all subdirectiries in a given directory. +// Input: (I1) dirPath - absolute path of a directory in question. +// Out: (O1) - slice of subdir asbolute paths; (O2) - error of the operation. +// Remark: If error (O2) is non-nil then (O1) is nil and vice versa. +func getSubdirAbsPaths(dirPath string) ([]string, error) { + sdns, err := getSubdirNames(dirPath) + if err != nil { + return nil, err + } + rsdns := []string{} + for _, sdn := range sdns { + rsdns = append(rsdns, filepath.Join(dirPath, sdn)) + } + return rsdns, nil +} + +// getOpenFilesInDir supplies a slice of os.File pointers to files located in the directory. +// Remark: Ignores files for which fileFilter returns false +func getOpenFilesInDir(dirPath string, fFilter fileFilter) ([]*os.File, error) { + dfi, err := os.Open(dirPath) + if err != nil { + return nil, newCannotOpenFileError("Cannot open directory " + dirPath) + } + defer dfi.Close() + // Size of read buffer (i.e. chunk of items read at a time). + rbs := 64 + resFiles := []*os.File{} +L: + for { + // Read directory entities by reasonable chuncks + // to prevent overflows on big number of files. + fis, e := dfi.Readdir(rbs) + switch e { + // It's OK. + case nil: + // Do nothing, just continue cycle. + case io.EOF: + break L + // Something went wrong. + default: + return nil, e + } + // THINK: Maybe, use async running. + for _, fi := range fis { + // NB: On Linux this could be a problem as + // there are lots of file types available. + if !fi.IsDir() { + f, e := os.Open(filepath.Join(dirPath, fi.Name())) + if e != nil { + if f != nil { + f.Close() + } + // THINK: Add nil as indicator that a problem occurred. + resFiles = append(resFiles, nil) + continue + } + // Check filter condition. + if fFilter != nil && !fFilter(fi, f) { + continue + } + resFiles = append(resFiles, f) + } + } + } + return resFiles, nil +} + +func isRegular(m os.FileMode) bool { + return m&os.ModeType == 0 +} + +// getDirFilePaths return full paths of the files located in the directory. +// Remark: Ignores files for which fileFilter returns false. +func getDirFilePaths(dirPath string, fpFilter filePathFilter, pathIsName bool) ([]string, error) { + dfi, err := os.Open(dirPath) + if err != nil { + return nil, newCannotOpenFileError("Cannot open directory " + dirPath) + } + defer dfi.Close() + + var absDirPath string + if !filepath.IsAbs(dirPath) { + absDirPath, err = filepath.Abs(dirPath) + if err != nil { + return nil, fmt.Errorf("cannot get absolute path of directory: %s", err.Error()) + } + } else { + absDirPath = dirPath + } + + // TODO: check if dirPath is really directory. + // Size of read buffer (i.e. chunk of items read at a time). + rbs := 2 << 5 + filePaths := []string{} + + var fp string +L: + for { + // Read directory entities by reasonable chuncks + // to prevent overflows on big number of files. + fis, e := dfi.Readdir(rbs) + switch e { + // It's OK. + case nil: + // Do nothing, just continue cycle. + case io.EOF: + break L + // Indicate that something went wrong. + default: + return nil, e + } + // THINK: Maybe, use async running. + for _, fi := range fis { + // NB: Should work on every Windows and non-Windows OS. + if isRegular(fi.Mode()) { + if pathIsName { + fp = fi.Name() + } else { + // Build full path of a file. + fp = filepath.Join(absDirPath, fi.Name()) + } + // Check filter condition. + if fpFilter != nil && !fpFilter(fp) { + continue + } + filePaths = append(filePaths, fp) + } + } + } + return filePaths, nil +} + +// getOpenFilesByDirectoryAsync runs async reading directories 'dirPaths' and inserts pairs +// in map 'filesInDirMap': Key - directory name, value - *os.File slice. +func getOpenFilesByDirectoryAsync( + dirPaths []string, + fFilter fileFilter, + filesInDirMap map[string][]*os.File, +) error { + n := len(dirPaths) + if n > maxDirNumberReadAsync { + return fmt.Errorf("number of input directories to be read exceeded max value %d", maxDirNumberReadAsync) + } + type filesInDirResult struct { + DirName string + Files []*os.File + Error error + } + dirFilesChan := make(chan *filesInDirResult, n) + var wg sync.WaitGroup + // Register n goroutines which are going to do work. + wg.Add(n) + for i := 0; i < n; i++ { + // Launch asynchronously the piece of work. + go func(dirPath string) { + fs, e := getOpenFilesInDir(dirPath, fFilter) + dirFilesChan <- &filesInDirResult{filepath.Base(dirPath), fs, e} + // Mark the current goroutine as finished (work is done). + wg.Done() + }(dirPaths[i]) + } + // Wait for all goroutines to finish their work. + wg.Wait() + // Close the error channel to let for-range clause + // get all the buffered values without blocking and quit in the end. + close(dirFilesChan) + for fidr := range dirFilesChan { + if fidr.Error == nil { + // THINK: What will happen if the key is already present? + filesInDirMap[fidr.DirName] = fidr.Files + } else { + return fidr.Error + } + } + return nil +} + +func copyFile(sf *os.File, dst string) (int64, error) { + df, err := os.Create(dst) + if err != nil { + return 0, err + } + defer df.Close() + return io.Copy(df, sf) +} + +// fileExists return flag whether a given file exists +// and operation error if an unclassified failure occurs. +func fileExists(path string) (bool, error) { + _, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// createDirectory makes directory with a given name +// making all parent directories if necessary. +func createDirectory(dirPath string) error { + var dPath string + var err error + if !filepath.IsAbs(dirPath) { + dPath, err = filepath.Abs(dirPath) + if err != nil { + return err + } + } else { + dPath = dirPath + } + exists, err := fileExists(dPath) + if err != nil { + return err + } + if exists { + return nil + } + return os.MkdirAll(dPath, os.ModeDir) +} + +// tryRemoveFile gives a try removing the file +// only ignoring an error when the file does not exist. +func tryRemoveFile(filePath string) (err error) { + err = os.Remove(filePath) + if os.IsNotExist(err) { + err = nil + return + } + return +} + +// Unzips a specified zip file. Returns filename->filebytes map. +func unzip(archiveName string) (map[string][]byte, error) { + // Open a zip archive for reading. + r, err := zip.OpenReader(archiveName) + if err != nil { + return nil, err + } + defer r.Close() + + // Files to be added to archive + // map file name to contents + files := make(map[string][]byte) + + // Iterate through the files in the archive, + // printing some of their contents. + for _, f := range r.File { + rc, err := f.Open() + if err != nil { + return nil, err + } + + bts, err := ioutil.ReadAll(rc) + rcErr := rc.Close() + + if err != nil { + return nil, err + } + if rcErr != nil { + return nil, rcErr + } + + files[f.Name] = bts + } + + return files, nil +} + +// Creates a zip file with the specified file names and byte contents. +func createZip(archiveName string, files map[string][]byte) error { + // Create a buffer to write our archive to. + buf := new(bytes.Buffer) + + // Create a new zip archive. + w := zip.NewWriter(buf) + + // Write files + for fpath, fcont := range files { + f, err := w.Create(fpath) + if err != nil { + return err + } + _, err = f.Write([]byte(fcont)) + if err != nil { + return err + } + } + + // Make sure to check the error on Close. + err := w.Close() + if err != nil { + return err + } + + err = ioutil.WriteFile(archiveName, buf.Bytes(), defaultFilePermissions) + if err != nil { + return err + } + + return nil +} diff --git a/lib/seelog/internals_xmlnode.go b/lib/seelog/internals_xmlnode.go new file mode 100644 index 000000000..985884933 --- /dev/null +++ b/lib/seelog/internals_xmlnode.go @@ -0,0 +1,175 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "encoding/xml" + "errors" + "fmt" + "io" + "strings" +) + +type xmlNode struct { + name string + attributes map[string]string + children []*xmlNode + value string +} + +func newNode() *xmlNode { + node := new(xmlNode) + node.children = make([]*xmlNode, 0) + node.attributes = make(map[string]string) + return node +} + +func (node *xmlNode) String() string { + str := fmt.Sprintf("<%s", node.name) + + for attrName, attrVal := range node.attributes { + str += fmt.Sprintf(" %s=\"%s\"", attrName, attrVal) + } + + str += ">" + str += node.value + + if len(node.children) != 0 { + for _, child := range node.children { + str += fmt.Sprintf("%s", child) + } + } + + str += fmt.Sprintf("", node.name) + + return str +} + +func (node *xmlNode) unmarshal(startEl xml.StartElement) error { + node.name = startEl.Name.Local + + for _, v := range startEl.Attr { + _, alreadyExists := node.attributes[v.Name.Local] + if alreadyExists { + return errors.New("tag '" + node.name + "' has duplicated attribute: '" + v.Name.Local + "'") + } + node.attributes[v.Name.Local] = v.Value + } + + return nil +} + +func (node *xmlNode) add(child *xmlNode) { + if node.children == nil { + node.children = make([]*xmlNode, 0) + } + + node.children = append(node.children, child) +} + +func (node *xmlNode) hasChildren() bool { + return node.children != nil && len(node.children) > 0 +} + +//============================================= + +func unmarshalConfig(reader io.Reader) (*xmlNode, error) { + xmlParser := xml.NewDecoder(reader) + + config, err := unmarshalNode(xmlParser, nil) + if err != nil { + return nil, err + } + if config == nil { + return nil, errors.New("xml has no content") + } + + nextConfigEntry, err := unmarshalNode(xmlParser, nil) + if nextConfigEntry != nil { + return nil, errors.New("xml contains more than one root element") + } + + return config, nil +} + +func unmarshalNode(xmlParser *xml.Decoder, curToken xml.Token) (node *xmlNode, err error) { + firstLoop := true + for { + var tok xml.Token + if firstLoop && curToken != nil { + tok = curToken + firstLoop = false + } else { + tok, err = getNextToken(xmlParser) + if err != nil || tok == nil { + return + } + } + + switch tt := tok.(type) { + case xml.SyntaxError: + err = errors.New(tt.Error()) + return + case xml.CharData: + value := strings.TrimSpace(string([]byte(tt))) + if node != nil { + node.value += value + } + case xml.StartElement: + if node == nil { + node = newNode() + err := node.unmarshal(tt) + if err != nil { + return nil, err + } + } else { + childNode, childErr := unmarshalNode(xmlParser, tok) + if childErr != nil { + return nil, childErr + } + + if childNode != nil { + node.add(childNode) + } else { + return + } + } + case xml.EndElement: + return + } + } +} + +func getNextToken(xmlParser *xml.Decoder) (tok xml.Token, err error) { + if tok, err = xmlParser.Token(); err != nil { + if err == io.EOF { + err = nil + return + } + return + } + + return +} diff --git a/lib/seelog/internals_xmlnode_test.go b/lib/seelog/internals_xmlnode_test.go new file mode 100644 index 000000000..9b9aa4348 --- /dev/null +++ b/lib/seelog/internals_xmlnode_test.go @@ -0,0 +1,196 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "strings" + "testing" + //"fmt" + "reflect" +) + +var testEnv *testing.T + +/*func TestWrapper(t *testing.T) { + testEnv = t + + s := "" + reader := strings.NewReader(s) + config, err := unmarshalConfig(reader) + if err != nil { + testEnv.Error(err) + return + } + + printXML(config, 0) +} + +func printXML(node *xmlNode, level int) { + indent := strings.Repeat("\t", level) + fmt.Print(indent + node.name) + for key, value := range node.attributes { + fmt.Print(" " + key + "/" + value) + } + fmt.Println() + + for _, child := range node.children { + printXML(child, level+1) + } +}*/ + +var xmlNodeTests []xmlNodeTest + +type xmlNodeTest struct { + testName string + inputXML string + expected interface{} + errorExpected bool +} + +func getXMLTests() []xmlNodeTest { + if xmlNodeTests == nil { + xmlNodeTests = make([]xmlNodeTest, 0) + + testName := "Simple test" + testXML := `` + testExpected := newNode() + testExpected.name = "a" + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, testExpected, false}) + + testName = "Multiline test" + testXML = + ` + + +` + testExpected = newNode() + testExpected.name = "a" + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, testExpected, false}) + + testName = "Multiline test #2" + testXML = + ` + + + + + + +` + testExpected = newNode() + testExpected.name = "a" + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, testExpected, false}) + + testName = "Incorrect names" + testXML = `< a >< /a >` + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, nil, true}) + + testName = "Comments" + testXML = + ` + + +` + testExpected = newNode() + testExpected.name = "a" + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, testExpected, false}) + + testName = "Multiple roots" + testXML = `` + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, nil, true}) + + testName = "Multiple roots + incorrect xml" + testXML = `` + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, nil, true}) + + testName = "Some unicode and data" + testXML = `<俄语>данные` + testExpected = newNode() + testExpected.name = "俄语" + testExpected.value = "данные" + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, testExpected, false}) + + testName = "Values and children" + testXML = `<俄语>данные` + testExpected = newNode() + testExpected.name = "俄语" + testExpected.value = "данные" + child := newNode() + child.name = "and_a_child" + testExpected.children = append(testExpected.children, child) + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, testExpected, false}) + + testName = "Just children" + testXML = `<俄语>` + testExpected = newNode() + testExpected.name = "俄语" + child = newNode() + child.name = "and_a_child" + testExpected.children = append(testExpected.children, child) + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, testExpected, false}) + + testName = "Mixed test" + testXML = `<俄语 a="1" b="2.13" c="abc">` + testExpected = newNode() + testExpected.name = "俄语" + testExpected.attributes["a"] = "1" + testExpected.attributes["b"] = "2.13" + testExpected.attributes["c"] = "abc" + child = newNode() + child.name = "child" + child.attributes["abc"] = "bca" + testExpected.children = append(testExpected.children, child) + child = newNode() + child.name = "child" + child.attributes["abc"] = "def" + testExpected.children = append(testExpected.children, child) + xmlNodeTests = append(xmlNodeTests, xmlNodeTest{testName, testXML, testExpected, false}) + } + + return xmlNodeTests +} + +func TestXmlNode(t *testing.T) { + + for _, test := range getXMLTests() { + + reader := strings.NewReader(test.inputXML) + parsedXML, err := unmarshalConfig(reader) + + if (err != nil) != test.errorExpected { + t.Errorf("\n%s:\nXML input: %s\nExpected error:%t. Got error: %t\n", test.testName, + test.inputXML, test.errorExpected, (err != nil)) + if err != nil { + t.Logf("%s\n", err.Error()) + } + continue + } + + if err == nil && !reflect.DeepEqual(parsedXML, test.expected) { + t.Errorf("\n%s:\nXML input: %s\nExpected: %s. \nGot: %s\n", test.testName, + test.inputXML, test.expected, parsedXML) + } + } +} diff --git a/lib/seelog/log.go b/lib/seelog/log.go new file mode 100644 index 000000000..5ae2c13cd --- /dev/null +++ b/lib/seelog/log.go @@ -0,0 +1,313 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "fmt" + "sync" + "time" +) + +const ( + staticFuncCallDepth = 3 // See 'commonLogger.log' method comments + loggerFuncCallDepth = 3 +) + +// Current is the logger used in all package level convenience funcs like 'Trace', 'Debug', 'Flush', etc. +var Current LoggerInterface + +// Default logger that is created from an empty config: "". It is not closed by a ReplaceLogger call. +var Default LoggerInterface + +// Disabled logger that doesn't produce any output in any circumstances. It is neither closed nor flushed by a ReplaceLogger call. +var Disabled LoggerInterface + +var pkgOperationsMutex *sync.Mutex + +func init() { + pkgOperationsMutex = new(sync.Mutex) + var err error + + if Default == nil { + Default, err = LoggerFromConfigAsBytes([]byte("")) + } + + if Disabled == nil { + Disabled, err = LoggerFromConfigAsBytes([]byte("")) + } + + if err != nil { + panic(fmt.Sprintf("Seelog couldn't start. Error: %s", err.Error())) + } + + Current = Default +} + +func createLoggerFromFullConfig(config *configForParsing) (LoggerInterface, error) { + if config.LogType == syncloggerTypeFromString { + return NewSyncLogger(&config.logConfig), nil + } else if config.LogType == asyncLooploggerTypeFromString { + return NewAsyncLoopLogger(&config.logConfig), nil + } else if config.LogType == asyncTimerloggerTypeFromString { + logData := config.LoggerData + if logData == nil { + return nil, errors.New("async timer data not set") + } + + asyncInt, ok := logData.(asyncTimerLoggerData) + if !ok { + return nil, errors.New("invalid async timer data") + } + + logger, err := NewAsyncTimerLogger(&config.logConfig, time.Duration(asyncInt.AsyncInterval)) + if !ok { + return nil, err + } + + return logger, nil + } else if config.LogType == adaptiveLoggerTypeFromString { + logData := config.LoggerData + if logData == nil { + return nil, errors.New("adaptive logger parameters not set") + } + + adaptData, ok := logData.(adaptiveLoggerData) + if !ok { + return nil, errors.New("invalid adaptive logger parameters") + } + + logger, err := NewAsyncAdaptiveLogger( + &config.logConfig, + time.Duration(adaptData.MinInterval), + time.Duration(adaptData.MaxInterval), + adaptData.CriticalMsgCount, + ) + if err != nil { + return nil, err + } + + return logger, nil + } + return nil, errors.New("invalid config log type/data") +} + +// UseLogger sets the 'Current' package level logger variable to the specified value. +// This variable is used in all Trace/Debug/... package level convenience funcs. +// +// Example: +// +// after calling +// +// seelog.UseLogger(somelogger) +// +// the following: +// +// seelog.Debug("abc") +// +// will be equal to +// +// somelogger.Debug("abc") +// +// IMPORTANT: UseLogger do NOT close the previous logger (only flushes it). So if +// you constantly use it to replace loggers and don't close them in other code, you'll +// end up having memory leaks. +// +// To safely replace loggers, use ReplaceLogger. +func UseLogger(logger LoggerInterface) error { + if logger == nil { + return errors.New("logger can not be nil") + } + + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + + oldLogger := Current + Current = logger + + if oldLogger != nil { + oldLogger.Flush() + } + + return nil +} + +// ReplaceLogger acts as UseLogger but the logger that was previously +// used is disposed (except Default and Disabled loggers). +// +// Example: +// +// import log "github.com/cihub/seelog" +// +// func main() { +// logger, err := log.LoggerFromConfigAsFile("seelog.xml") +// +// if err != nil { +// panic(err) +// } +// +// log.ReplaceLogger(logger) +// defer log.Flush() +// +// log.Trace("test") +// log.Debugf("var = %s", "abc") +// } +func ReplaceLogger(logger LoggerInterface) error { + if logger == nil { + return errors.New("logger can not be nil") + } + + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + + defer func() { + if err := recover(); err != nil { + reportInternalError(fmt.Errorf("recovered from panic during ReplaceLogger: %s", err)) + } + }() + + if Current == Default { + Current.Flush() + } else if Current != nil && !Current.Closed() && Current != Disabled { + Current.Flush() + Current.Close() + } + + Current = logger + + return nil +} + +// Tracef formats message according to format specifier +// and writes to default logger with log level = Trace. +func Tracef(format string, params ...interface{}) { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + Current.traceWithCallDepth(staticFuncCallDepth, newLogFormattedMessage(format, params)) +} + +// Debugf formats message according to format specifier +// and writes to default logger with log level = Debug. +func Debugf(format string, params ...interface{}) { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + Current.debugWithCallDepth(staticFuncCallDepth, newLogFormattedMessage(format, params)) +} + +// Infof formats message according to format specifier +// and writes to default logger with log level = Info. +func Infof(format string, params ...interface{}) { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + Current.infoWithCallDepth(staticFuncCallDepth, newLogFormattedMessage(format, params)) +} + +// Warnf formats message according to format specifier and writes to default logger with log level = Warn +func Warnf(format string, params ...interface{}) error { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + message := newLogFormattedMessage(format, params) + Current.warnWithCallDepth(staticFuncCallDepth, message) + return errors.New(message.String()) +} + +// Errorf formats message according to format specifier and writes to default logger with log level = Error +func Errorf(format string, params ...interface{}) error { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + message := newLogFormattedMessage(format, params) + Current.errorWithCallDepth(staticFuncCallDepth, message) + return errors.New(message.String()) +} + +// Criticalf formats message according to format specifier and writes to default logger with log level = Critical +func Criticalf(format string, params ...interface{}) error { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + message := newLogFormattedMessage(format, params) + Current.criticalWithCallDepth(staticFuncCallDepth, message) + return errors.New(message.String()) +} + +// Trace formats message using the default formats for its operands and writes to default logger with log level = Trace +func Trace(v ...interface{}) { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + Current.traceWithCallDepth(staticFuncCallDepth, newLogMessage(v)) +} + +// Debug formats message using the default formats for its operands and writes to default logger with log level = Debug +func Debug(v ...interface{}) { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + Current.debugWithCallDepth(staticFuncCallDepth, newLogMessage(v)) +} + +// Info formats message using the default formats for its operands and writes to default logger with log level = Info +func Info(v ...interface{}) { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + Current.infoWithCallDepth(staticFuncCallDepth, newLogMessage(v)) +} + +// Warn formats message using the default formats for its operands and writes to default logger with log level = Warn +func Warn(v ...interface{}) error { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + message := newLogMessage(v) + Current.warnWithCallDepth(staticFuncCallDepth, message) + return errors.New(message.String()) +} + +// Error formats message using the default formats for its operands and writes to default logger with log level = Error +func Error(v ...interface{}) error { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + message := newLogMessage(v) + Current.errorWithCallDepth(staticFuncCallDepth, message) + return errors.New(message.String()) +} + +// Critical formats message using the default formats for its operands and writes to default logger with log level = Critical +func Critical(v ...interface{}) error { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + message := newLogMessage(v) + Current.criticalWithCallDepth(staticFuncCallDepth, message) + return errors.New(message.String()) +} + +// Flush immediately processes all currently queued messages and all currently buffered messages. +// It is a blocking call which returns only after the queue is empty and all the buffers are empty. +// +// If Flush is called for a synchronous logger (type='sync'), it only flushes buffers (e.g. '' receivers) +// , because there is no queue. +// +// Call this method when your app is going to shut down not to lose any log messages. +func Flush() { + pkgOperationsMutex.Lock() + defer pkgOperationsMutex.Unlock() + Current.Flush() +} diff --git a/lib/seelog/logger.go b/lib/seelog/logger.go new file mode 100644 index 000000000..ed714ab21 --- /dev/null +++ b/lib/seelog/logger.go @@ -0,0 +1,370 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "fmt" + "sync" +) + +func reportInternalError(err error) { + //fmt.Fprintf(os.Stderr, "seelog internal error: %s\n", err) + //panic(err) +} + +// LoggerInterface represents structs capable of logging Seelog messages +type LoggerInterface interface { + + // Tracef formats message according to format specifier + // and writes to log with level = Trace. + Tracef(format string, params ...interface{}) + + // Debugf formats message according to format specifier + // and writes to log with level = Debug. + Debugf(format string, params ...interface{}) + + // Infof formats message according to format specifier + // and writes to log with level = Info. + Infof(format string, params ...interface{}) + + // Warnf formats message according to format specifier + // and writes to log with level = Warn. + Warnf(format string, params ...interface{}) error + + // Errorf formats message according to format specifier + // and writes to log with level = Error. + Errorf(format string, params ...interface{}) error + + // Criticalf formats message according to format specifier + // and writes to log with level = Critical. + Criticalf(format string, params ...interface{}) error + + // Trace formats message using the default formats for its operands + // and writes to log with level = Trace + Trace(v ...interface{}) + + // Debug formats message using the default formats for its operands + // and writes to log with level = Debug + Debug(v ...interface{}) + + // Info formats message using the default formats for its operands + // and writes to log with level = Info + Info(v ...interface{}) + + // Warn formats message using the default formats for its operands + // and writes to log with level = Warn + Warn(v ...interface{}) error + + // Error formats message using the default formats for its operands + // and writes to log with level = Error + Error(v ...interface{}) error + + // Critical formats message using the default formats for its operands + // and writes to log with level = Critical + Critical(v ...interface{}) error + + traceWithCallDepth(callDepth int, message fmt.Stringer) + debugWithCallDepth(callDepth int, message fmt.Stringer) + infoWithCallDepth(callDepth int, message fmt.Stringer) + warnWithCallDepth(callDepth int, message fmt.Stringer) + errorWithCallDepth(callDepth int, message fmt.Stringer) + criticalWithCallDepth(callDepth int, message fmt.Stringer) + + // Close flushes all the messages in the logger and closes it. It cannot be used after this operation. + Close() + + // Flush flushes all the messages in the logger. + Flush() + + // Closed returns true if the logger was previously closed. + Closed() bool + + // SetAdditionalStackDepth sets the additional number of frames to skip by runtime.Caller + // when getting function information needed to print seelog format identifiers such as %Func or %File. + // + // This func may be used when you wrap seelog funcs and want to print caller info of you own + // wrappers instead of seelog func callers. In this case you should set depth = 1. If you then + // wrap your wrapper, you should set depth = 2, etc. + // + // NOTE: Incorrect depth value may lead to errors in runtime.Caller evaluation or incorrect + // function/file names in log files. Do not use it if you are not going to wrap seelog funcs. + // You may reset the value to default using a SetAdditionalStackDepth(0) call. + SetAdditionalStackDepth(depth int) error + + // Sets logger context that can be used in formatter funcs and custom receivers + SetContext(context interface{}) +} + +// innerLoggerInterface is an internal logging interface +type innerLoggerInterface interface { + innerLog(level LogLevel, context LogContextInterface, message fmt.Stringer) + Flush() +} + +// [file path][func name][level] -> [allowed] +type allowedContextCache map[string]map[string]map[LogLevel]bool + +// commonLogger contains all common data needed for logging and contains methods used to log messages. +type commonLogger struct { + config *logConfig // Config used for logging + contextCache allowedContextCache // Caches whether log is enabled for specific "full path-func name-level" sets + closed bool // 'true' when all writers are closed, all data is flushed, logger is unusable. Must be accessed while holding closedM + closedM sync.RWMutex + m sync.Mutex // Mutex for main operations + unusedLevels []bool + innerLogger innerLoggerInterface + addStackDepth int // Additional stack depth needed for correct seelog caller context detection + customContext interface{} +} + +func newCommonLogger(config *logConfig, internalLogger innerLoggerInterface) *commonLogger { + cLogger := new(commonLogger) + + cLogger.config = config + cLogger.contextCache = make(allowedContextCache) + cLogger.unusedLevels = make([]bool, Off) + cLogger.fillUnusedLevels() + cLogger.innerLogger = internalLogger + + return cLogger +} + +func (cLogger *commonLogger) SetAdditionalStackDepth(depth int) error { + if depth < 0 { + return fmt.Errorf("negative depth: %d", depth) + } + cLogger.m.Lock() + cLogger.addStackDepth = depth + cLogger.m.Unlock() + return nil +} + +func (cLogger *commonLogger) Tracef(format string, params ...interface{}) { + cLogger.traceWithCallDepth(loggerFuncCallDepth, newLogFormattedMessage(format, params)) +} + +func (cLogger *commonLogger) Debugf(format string, params ...interface{}) { + cLogger.debugWithCallDepth(loggerFuncCallDepth, newLogFormattedMessage(format, params)) +} + +func (cLogger *commonLogger) Infof(format string, params ...interface{}) { + cLogger.infoWithCallDepth(loggerFuncCallDepth, newLogFormattedMessage(format, params)) +} + +func (cLogger *commonLogger) Warnf(format string, params ...interface{}) error { + message := newLogFormattedMessage(format, params) + cLogger.warnWithCallDepth(loggerFuncCallDepth, message) + return errors.New(message.String()) +} + +func (cLogger *commonLogger) Errorf(format string, params ...interface{}) error { + message := newLogFormattedMessage(format, params) + cLogger.errorWithCallDepth(loggerFuncCallDepth, message) + return errors.New(message.String()) +} + +func (cLogger *commonLogger) Criticalf(format string, params ...interface{}) error { + message := newLogFormattedMessage(format, params) + cLogger.criticalWithCallDepth(loggerFuncCallDepth, message) + return errors.New(message.String()) +} + +func (cLogger *commonLogger) Trace(v ...interface{}) { + cLogger.traceWithCallDepth(loggerFuncCallDepth, newLogMessage(v)) +} + +func (cLogger *commonLogger) Debug(v ...interface{}) { + cLogger.debugWithCallDepth(loggerFuncCallDepth, newLogMessage(v)) +} + +func (cLogger *commonLogger) Info(v ...interface{}) { + cLogger.infoWithCallDepth(loggerFuncCallDepth, newLogMessage(v)) +} + +func (cLogger *commonLogger) Warn(v ...interface{}) error { + message := newLogMessage(v) + cLogger.warnWithCallDepth(loggerFuncCallDepth, message) + return errors.New(message.String()) +} + +func (cLogger *commonLogger) Error(v ...interface{}) error { + message := newLogMessage(v) + cLogger.errorWithCallDepth(loggerFuncCallDepth, message) + return errors.New(message.String()) +} + +func (cLogger *commonLogger) Critical(v ...interface{}) error { + message := newLogMessage(v) + cLogger.criticalWithCallDepth(loggerFuncCallDepth, message) + return errors.New(message.String()) +} + +func (cLogger *commonLogger) SetContext(c interface{}) { + cLogger.customContext = c +} + +func (cLogger *commonLogger) traceWithCallDepth(callDepth int, message fmt.Stringer) { + cLogger.log(TraceLvl, message, callDepth) +} + +func (cLogger *commonLogger) debugWithCallDepth(callDepth int, message fmt.Stringer) { + cLogger.log(DebugLvl, message, callDepth) +} + +func (cLogger *commonLogger) infoWithCallDepth(callDepth int, message fmt.Stringer) { + cLogger.log(InfoLvl, message, callDepth) +} + +func (cLogger *commonLogger) warnWithCallDepth(callDepth int, message fmt.Stringer) { + cLogger.log(WarnLvl, message, callDepth) +} + +func (cLogger *commonLogger) errorWithCallDepth(callDepth int, message fmt.Stringer) { + cLogger.log(ErrorLvl, message, callDepth) +} + +func (cLogger *commonLogger) criticalWithCallDepth(callDepth int, message fmt.Stringer) { + cLogger.log(CriticalLvl, message, callDepth) + cLogger.innerLogger.Flush() +} + +func (cLogger *commonLogger) Closed() bool { + cLogger.closedM.RLock() + defer cLogger.closedM.RUnlock() + return cLogger.closed +} + +func (cLogger *commonLogger) fillUnusedLevels() { + for i := 0; i < len(cLogger.unusedLevels); i++ { + cLogger.unusedLevels[i] = true + } + + cLogger.fillUnusedLevelsByContraint(cLogger.config.Constraints) + + for _, exception := range cLogger.config.Exceptions { + cLogger.fillUnusedLevelsByContraint(exception) + } +} + +func (cLogger *commonLogger) fillUnusedLevelsByContraint(constraint logLevelConstraints) { + for i := 0; i < len(cLogger.unusedLevels); i++ { + if constraint.IsAllowed(LogLevel(i)) { + cLogger.unusedLevels[i] = false + } + } +} + +// stackCallDepth is used to indicate the call depth of 'log' func. +// This depth level is used in the runtime.Caller(...) call. See +// common_context.go -> specifyContext, extractCallerInfo for details. +func (cLogger *commonLogger) log(level LogLevel, message fmt.Stringer, stackCallDepth int) { + if cLogger.unusedLevels[level] { + return + } + cLogger.m.Lock() + defer cLogger.m.Unlock() + + if cLogger.Closed() { + return + } + context, _ := specifyContext(stackCallDepth+cLogger.addStackDepth, cLogger.customContext) + // Context errors are not reported because there are situations + // in which context errors are normal Seelog usage cases. For + // example in executables with stripped symbols. + // Error contexts are returned instead. See common_context.go. + /*if err != nil { + reportInternalError(err) + return + }*/ + cLogger.innerLogger.innerLog(level, context, message) +} + +func (cLogger *commonLogger) processLogMsg(level LogLevel, message fmt.Stringer, context LogContextInterface) { + defer func() { + if err := recover(); err != nil { + reportInternalError(fmt.Errorf("recovered from panic during message processing: %s, %v %v %v", err, level, message, context)) + } + }() + if cLogger.config.IsAllowed(level, context) { + cLogger.config.RootDispatcher.Dispatch(message.String(), level, context, reportInternalError) + } +} + +func (cLogger *commonLogger) isAllowed(level LogLevel, context LogContextInterface) bool { + funcMap, ok := cLogger.contextCache[context.FullPath()] + if !ok { + funcMap = make(map[string]map[LogLevel]bool, 0) + cLogger.contextCache[context.FullPath()] = funcMap + } + + levelMap, ok := funcMap[context.Func()] + if !ok { + levelMap = make(map[LogLevel]bool, 0) + funcMap[context.Func()] = levelMap + } + + isAllowValue, ok := levelMap[level] + if !ok { + isAllowValue = cLogger.config.IsAllowed(level, context) + levelMap[level] = isAllowValue + } + + return isAllowValue +} + +type logMessage struct { + params []interface{} +} + +type logFormattedMessage struct { + format string + params []interface{} +} + +func newLogMessage(params []interface{}) fmt.Stringer { + message := new(logMessage) + + message.params = params + + return message +} + +func newLogFormattedMessage(format string, params []interface{}) *logFormattedMessage { + message := new(logFormattedMessage) + + message.params = params + message.format = format + + return message +} + +func (message *logMessage) String() string { + return fmt.Sprint(message.params...) +} + +func (message *logFormattedMessage) String() string { + return fmt.Sprintf(message.format, message.params...) +} diff --git a/lib/seelog/writers_bufferedwriter.go b/lib/seelog/writers_bufferedwriter.go new file mode 100644 index 000000000..caca55fdf --- /dev/null +++ b/lib/seelog/writers_bufferedwriter.go @@ -0,0 +1,178 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "bufio" + "errors" + "fmt" + "io" + "sync" + "time" +) + +// bufferedWriter stores data in memory and flushes it every flushPeriod or when buffer is full +type bufferedWriter struct { + flushPeriod time.Duration // data flush interval (converted from configured milliseconds) + bufferMutex *sync.Mutex // mutex for buffer operations synchronization + innerWriter io.Writer // inner writer + buffer *bufio.Writer // buffered wrapper for inner writer + bufferSize int // max size of data chunk in bytes + done chan struct{} // stop signal for periodic flushing goroutine + closeOnce sync.Once + closeErr error +} + +// NewBufferedWriter creates a new buffered writer struct. +// bufferSize -- size of memory buffer in bytes +// flushPeriod -- period in which data flushes from memory buffer in milliseconds. 0 - turn off this functionality +func NewBufferedWriter(innerWriter io.Writer, bufferSize int, flushPeriod time.Duration) (*bufferedWriter, error) { + + if innerWriter == nil { + return nil, errors.New("argument is nil: innerWriter") + } + if flushPeriod < 0 { + return nil, fmt.Errorf("flushPeriod can not be less than 0. Got: %d", flushPeriod) + } + + if bufferSize <= 0 { + return nil, fmt.Errorf("bufferSize can not be less or equal to 0. Got: %d", bufferSize) + } + + buffer := bufio.NewWriterSize(innerWriter, bufferSize) + + /*if err != nil { + return nil, err + }*/ + + newWriter := new(bufferedWriter) + + newWriter.innerWriter = innerWriter + newWriter.buffer = buffer + newWriter.bufferSize = bufferSize + newWriter.flushPeriod = flushPeriod * time.Millisecond + newWriter.bufferMutex = new(sync.Mutex) + newWriter.done = make(chan struct{}) + + if flushPeriod != 0 { + go newWriter.flushPeriodically() + } + + return newWriter, nil +} + +func (bufWriter *bufferedWriter) writeBigChunk(bytes []byte) (n int, err error) { + bufferedLen := bufWriter.buffer.Buffered() + + n, err = bufWriter.flushInner() + if err != nil { + return + } + + written, writeErr := bufWriter.innerWriter.Write(bytes) + return bufferedLen + written, writeErr +} + +// Sends data to buffer manager. Waits until all buffers are full. +func (bufWriter *bufferedWriter) Write(bytes []byte) (n int, err error) { + + bufWriter.bufferMutex.Lock() + defer bufWriter.bufferMutex.Unlock() + + bytesLen := len(bytes) + + if bytesLen > bufWriter.bufferSize { + return bufWriter.writeBigChunk(bytes) + } + + if bytesLen > bufWriter.buffer.Available() { + n, err = bufWriter.flushInner() + if err != nil { + return + } + } + + bufWriter.buffer.Write(bytes) + + return len(bytes), nil +} + +func (bufWriter *bufferedWriter) Close() error { + bufWriter.closeOnce.Do(func() { + close(bufWriter.done) + bufWriter.Flush() + + closer, ok := bufWriter.innerWriter.(io.Closer) + if ok { + bufWriter.closeErr = closer.Close() + } + }) + + return bufWriter.closeErr +} + +func (bufWriter *bufferedWriter) Flush() { + + bufWriter.bufferMutex.Lock() + defer bufWriter.bufferMutex.Unlock() + + bufWriter.flushInner() +} + +func (bufWriter *bufferedWriter) flushInner() (n int, err error) { + bufferedLen := bufWriter.buffer.Buffered() + flushErr := bufWriter.buffer.Flush() + + return bufWriter.buffer.Buffered() - bufferedLen, flushErr +} + +func (bufWriter *bufferedWriter) flushBuffer() { + bufWriter.bufferMutex.Lock() + defer bufWriter.bufferMutex.Unlock() + + bufWriter.buffer.Flush() +} + +func (bufWriter *bufferedWriter) flushPeriodically() { + if bufWriter.flushPeriod <= 0 { + return + } + + ticker := time.NewTicker(bufWriter.flushPeriod) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + bufWriter.flushBuffer() + case <-bufWriter.done: + return + } + } +} + +func (bufWriter *bufferedWriter) String() string { + return fmt.Sprintf("bufferedWriter size: %d, flushPeriod: %d", bufWriter.bufferSize, bufWriter.flushPeriod) +} diff --git a/lib/seelog/writers_bufferedwriter_test.go b/lib/seelog/writers_bufferedwriter_test.go new file mode 100644 index 000000000..d50465c05 --- /dev/null +++ b/lib/seelog/writers_bufferedwriter_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "testing" + "time" +) + +func TestChunkWriteOnFilling(t *testing.T) { + writer, _ := newBytesVerifier(t) + bufferedWriter, err := NewBufferedWriter(writer, 1024, 0) + + if err != nil { + t.Fatalf("Unexpected buffered writer creation error: %s", err.Error()) + } + + bytes := make([]byte, 1000) + + bufferedWriter.Write(bytes) + writer.ExpectBytes(bytes) + bufferedWriter.Write(bytes) +} + +func TestFlushByTimePeriod(t *testing.T) { + writer, _ := newBytesVerifier(t) + bufferedWriter, err := NewBufferedWriter(writer, 1024, 10) + + if err != nil { + t.Fatalf("Unexpected buffered writer creation error: %s", err.Error()) + } + defer bufferedWriter.Close() + + bytes := []byte("Hello") + + for i := 0; i < 2; i++ { + writer.ExpectBytes(bytes) + bufferedWriter.Write(bytes) + waitForFlush(t, writer, time.Second) + } +} + +func waitForFlush(t *testing.T, writer *bytesVerifier, timeout time.Duration) { + t.Helper() + + deadline := time.Now().Add(timeout) + for writer.waitingForInput && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + + if writer.waitingForInput { + t.Fatal("timed out waiting for periodic flush") + } +} + +func TestBigMessageMustPassMemoryBuffer(t *testing.T) { + writer, _ := newBytesVerifier(t) + bufferedWriter, err := NewBufferedWriter(writer, 1024, 0) + + if err != nil { + t.Fatalf("Unexpected buffered writer creation error: %s", err.Error()) + } + + bytes := make([]byte, 5000) + + for i := 0; i < len(bytes); i++ { + bytes[i] = uint8(i % 255) + } + + writer.ExpectBytes(bytes) + bufferedWriter.Write(bytes) +} diff --git a/lib/seelog/writers_connwriter.go b/lib/seelog/writers_connwriter.go new file mode 100644 index 000000000..d199894e7 --- /dev/null +++ b/lib/seelog/writers_connwriter.go @@ -0,0 +1,144 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "crypto/tls" + "fmt" + "io" + "net" +) + +// connWriter is used to write to a stream-oriented network connection. +type connWriter struct { + innerWriter io.WriteCloser + reconnectOnMsg bool + reconnect bool + net string + addr string + useTLS bool + configTLS *tls.Config +} + +// Creates writer to the address addr on the network netName. +// Connection will be opened on each write if reconnectOnMsg = true +func NewConnWriter(netName string, addr string, reconnectOnMsg bool) *connWriter { + newWriter := new(connWriter) + + newWriter.net = netName + newWriter.addr = addr + newWriter.reconnectOnMsg = reconnectOnMsg + + return newWriter +} + +// Creates a writer that uses SSL/TLS +func newTLSWriter(netName string, addr string, reconnectOnMsg bool, config *tls.Config) *connWriter { + newWriter := new(connWriter) + + newWriter.net = netName + newWriter.addr = addr + newWriter.reconnectOnMsg = reconnectOnMsg + newWriter.useTLS = true + newWriter.configTLS = config + + return newWriter +} + +func (connWriter *connWriter) Close() error { + if connWriter.innerWriter == nil { + return nil + } + + return connWriter.innerWriter.Close() +} + +func (connWriter *connWriter) Write(bytes []byte) (n int, err error) { + if connWriter.neededConnectOnMsg() { + err = connWriter.connect() + if err != nil { + return 0, err + } + } + + if connWriter.reconnectOnMsg { + defer connWriter.innerWriter.Close() + } + + n, err = connWriter.innerWriter.Write(bytes) + if err != nil { + connWriter.reconnect = true + } + + return +} + +func (connWriter *connWriter) String() string { + return fmt.Sprintf("Conn writer: [%s, %s, %v]", connWriter.net, connWriter.addr, connWriter.reconnectOnMsg) +} + +func (connWriter *connWriter) connect() error { + if connWriter.innerWriter != nil { + connWriter.innerWriter.Close() + connWriter.innerWriter = nil + } + + if connWriter.useTLS { + conn, err := tls.Dial(connWriter.net, connWriter.addr, connWriter.configTLS) + if err != nil { + return err + } + connWriter.innerWriter = conn + + return nil + } + + conn, err := net.Dial(connWriter.net, connWriter.addr) + if err != nil { + return err + } + + tcpConn, ok := conn.(*net.TCPConn) + if ok { + tcpConn.SetKeepAlive(true) + } + + connWriter.innerWriter = conn + + return nil +} + +func (connWriter *connWriter) neededConnectOnMsg() bool { + if connWriter.reconnect { + connWriter.reconnect = false + return true + } + + if connWriter.innerWriter == nil { + return true + } + + return connWriter.reconnectOnMsg +} diff --git a/lib/seelog/writers_consolewriter.go b/lib/seelog/writers_consolewriter.go new file mode 100644 index 000000000..3eb79afa9 --- /dev/null +++ b/lib/seelog/writers_consolewriter.go @@ -0,0 +1,47 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import "fmt" + +// consoleWriter is used to write to console +type consoleWriter struct { +} + +// Creates a new console writer. Returns error, if the console writer couldn't be created. +func NewConsoleWriter() (writer *consoleWriter, err error) { + newWriter := new(consoleWriter) + + return newWriter, nil +} + +// Create folder and file on WriteLog/Write first call +func (console *consoleWriter) Write(bytes []byte) (int, error) { + return fmt.Print(string(bytes)) +} + +func (console *consoleWriter) String() string { + return "Console writer" +} diff --git a/lib/seelog/writers_filewriter.go b/lib/seelog/writers_filewriter.go new file mode 100644 index 000000000..8d3ae270e --- /dev/null +++ b/lib/seelog/writers_filewriter.go @@ -0,0 +1,92 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" + "io" + "os" + "path/filepath" +) + +// fileWriter is used to write to a file. +type fileWriter struct { + innerWriter io.WriteCloser + fileName string +} + +// Creates a new file and a corresponding writer. Returns error, if the file couldn't be created. +func NewFileWriter(fileName string) (writer *fileWriter, err error) { + newWriter := new(fileWriter) + newWriter.fileName = fileName + + return newWriter, nil +} + +func (fw *fileWriter) Close() error { + if fw.innerWriter != nil { + err := fw.innerWriter.Close() + if err != nil { + return err + } + fw.innerWriter = nil + } + return nil +} + +// Create folder and file on WriteLog/Write first call +func (fw *fileWriter) Write(bytes []byte) (n int, err error) { + if fw.innerWriter == nil { + if err := fw.createFile(); err != nil { + return 0, err + } + } + return fw.innerWriter.Write(bytes) +} + +func (fw *fileWriter) createFile() error { + folder, _ := filepath.Split(fw.fileName) + var err error + + if 0 != len(folder) { + err = os.MkdirAll(folder, defaultDirectoryPermissions) + if err != nil { + return err + } + } + + // If exists + fw.innerWriter, err = os.OpenFile(fw.fileName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, defaultFilePermissions) + + if err != nil { + return err + } + + return nil +} + +func (fw *fileWriter) String() string { + return fmt.Sprintf("File writer: %s", fw.fileName) +} diff --git a/lib/seelog/writers_filewriter_test.go b/lib/seelog/writers_filewriter_test.go new file mode 100644 index 000000000..7d2710746 --- /dev/null +++ b/lib/seelog/writers_filewriter_test.go @@ -0,0 +1,254 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +const ( + messageLen = 10 +) + +var bytesFileTest = []byte(strings.Repeat("A", messageLen)) + +func TestSimpleFileWriter(t *testing.T) { + t.Logf("Starting file writer tests") + NewFileWriterTester(simplefileWriterTests, simplefileWriterGetter, t).test() +} + +//=============================================================== + +func simplefileWriterGetter(testCase *fileWriterTestCase) (io.WriteCloser, error) { + return NewFileWriter(testCase.fileName) +} + +// =============================================================== +type fileWriterTestCase struct { + files []string + fileName string + rollingType rollingType + fileSize int64 + maxRolls int + datePattern string + writeCount int + resFiles []string + nameMode rollingNameMode +} + +func createSimplefileWriterTestCase(fileName string, writeCount int) *fileWriterTestCase { + return &fileWriterTestCase{[]string{}, fileName, rollingTypeSize, 0, 0, "", writeCount, []string{fileName}, 0} +} + +var simplefileWriterTests = []*fileWriterTestCase{ + createSimplefileWriterTestCase("log.testlog", 1), + createSimplefileWriterTestCase("log.testlog", 50), + createSimplefileWriterTestCase(filepath.Join("dir", "log.testlog"), 50), +} + +//=============================================================== + +type fileWriterTester struct { + testCases []*fileWriterTestCase + writerGetter func(*fileWriterTestCase) (io.WriteCloser, error) + t *testing.T +} + +func NewFileWriterTester( + testCases []*fileWriterTestCase, + writerGetter func(*fileWriterTestCase) (io.WriteCloser, error), + t *testing.T) *fileWriterTester { + + return &fileWriterTester{testCases, writerGetter, t} +} + +func isWriterTestFile(fn string) bool { + return strings.Contains(fn, ".testlog") +} + +func cleanupWriterTest(t *testing.T) { + toDel, err := getDirFilePaths(".", isWriterTestFile, true) + if nil != err { + t.Fatal("Cannot list files in test directory!") + } + + for _, p := range toDel { + if err = tryRemoveFile(p); nil != err { + t.Errorf("cannot remove file %s in test directory: %s", p, err.Error()) + } + } + + if err = os.RemoveAll("dir"); nil != err { + t.Errorf("cannot remove temp test directory: %s", err.Error()) + } +} + +func getWriterTestResultFiles() ([]string, error) { + var p []string + + visit := func(path string, f os.FileInfo, err error) error { + if !f.IsDir() && isWriterTestFile(path) { + abs, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("filepath.Abs failed for %s", path) + } + + p = append(p, abs) + } + + return nil + } + + err := filepath.Walk(".", visit) + if nil != err { + return nil, err + } + + return p, nil +} + +func (tester *fileWriterTester) testCase(testCase *fileWriterTestCase, testNum int) { + defer cleanupWriterTest(tester.t) + + tester.t.Logf("Start test [%v]\n", testNum) + + for _, filePath := range testCase.files { + dir, _ := filepath.Split(filePath) + + var err error + + if 0 != len(dir) { + err = os.MkdirAll(dir, defaultDirectoryPermissions) + if err != nil { + tester.t.Error(err) + return + } + } + + fi, err := os.Create(filePath) + if err != nil { + tester.t.Error(err) + return + } + + err = fi.Close() + if err != nil { + tester.t.Error(err) + return + } + } + + fwc, err := tester.writerGetter(testCase) + if err != nil { + tester.t.Error(err) + return + } + defer fwc.Close() + + tester.performWrite(fwc, testCase.writeCount) + + files, err := getWriterTestResultFiles() + if err != nil { + tester.t.Error(err) + return + } + + tester.checkRequiredFilesExist(testCase, files) + tester.checkJustRequiredFilesExist(testCase, files) + +} + +func (tester *fileWriterTester) test() { + for i, tc := range tester.testCases { + cleanupWriterTest(tester.t) + tester.testCase(tc, i) + } +} + +func (tester *fileWriterTester) performWrite(fileWriter io.Writer, count int) { + for i := 0; i < count; i++ { + _, err := fileWriter.Write(bytesFileTest) + + if err != nil { + tester.t.Error(err) + return + } + } +} + +func (tester *fileWriterTester) checkRequiredFilesExist(testCase *fileWriterTestCase, files []string) { + var found bool + for _, expected := range testCase.resFiles { + found = false + exAbs, err := filepath.Abs(expected) + if err != nil { + tester.t.Errorf("filepath.Abs failed for %s", expected) + continue + } + + for _, f := range files { + if af, e := filepath.Abs(f); e == nil { + tester.t.Log(af) + if exAbs == af { + found = true + break + } + } else { + tester.t.Errorf("filepath.Abs failed for %s", f) + } + } + + if !found { + tester.t.Errorf("expected file: %s doesn't exist. Got %v\n", exAbs, files) + } + } +} + +func (tester *fileWriterTester) checkJustRequiredFilesExist(testCase *fileWriterTestCase, files []string) { + for _, f := range files { + found := false + for _, expected := range testCase.resFiles { + + exAbs, err := filepath.Abs(expected) + if err != nil { + tester.t.Errorf("filepath.Abs failed for %s", expected) + } else { + if exAbs == f { + found = true + break + } + } + } + + if !found { + tester.t.Errorf("unexpected file: %v", f) + } + } +} diff --git a/lib/seelog/writers_formattedwriter.go b/lib/seelog/writers_formattedwriter.go new file mode 100644 index 000000000..bf44a4103 --- /dev/null +++ b/lib/seelog/writers_formattedwriter.go @@ -0,0 +1,62 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "errors" + "fmt" + "io" +) + +type formattedWriter struct { + writer io.Writer + formatter *formatter +} + +func NewFormattedWriter(writer io.Writer, formatter *formatter) (*formattedWriter, error) { + if formatter == nil { + return nil, errors.New("formatter can not be nil") + } + + return &formattedWriter{writer, formatter}, nil +} + +func (formattedWriter *formattedWriter) Write(message string, level LogLevel, context LogContextInterface) error { + str := formattedWriter.formatter.Format(message, level, context) + _, err := formattedWriter.writer.Write([]byte(str)) + return err +} + +func (formattedWriter *formattedWriter) String() string { + return fmt.Sprintf("writer: %s, format: %s", formattedWriter.writer, formattedWriter.formatter) +} + +func (formattedWriter *formattedWriter) Writer() io.Writer { + return formattedWriter.writer +} + +func (formattedWriter *formattedWriter) Format() *formatter { + return formattedWriter.formatter +} diff --git a/lib/seelog/writers_formattedwriter_test.go b/lib/seelog/writers_formattedwriter_test.go new file mode 100644 index 000000000..351ac4eff --- /dev/null +++ b/lib/seelog/writers_formattedwriter_test.go @@ -0,0 +1,65 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "testing" +) + +func TestformattedWriter(t *testing.T) { + formatStr := "%Level %LEVEL %Msg" + message := "message" + var logLevel = LogLevel(TraceLvl) + + bytesVerifier, err := newBytesVerifier(t) + if err != nil { + t.Error(err) + return + } + + formatter, err := NewFormatter(formatStr) + if err != nil { + t.Error(err) + return + } + + writer, err := NewFormattedWriter(bytesVerifier, formatter) + if err != nil { + t.Error(err) + return + } + + context, err := currentContext(nil) + if err != nil { + t.Error(err) + return + } + + logMessage := formatter.Format(message, logLevel, context) + + bytesVerifier.ExpectBytes([]byte(logMessage)) + writer.Write(message, logLevel, context) + bytesVerifier.MustNotExpect() +} diff --git a/lib/seelog/writers_rollingfilewriter.go b/lib/seelog/writers_rollingfilewriter.go new file mode 100644 index 000000000..2422a67cf --- /dev/null +++ b/lib/seelog/writers_rollingfilewriter.go @@ -0,0 +1,625 @@ +// Copyright (c) 2013 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +// Common constants +const ( + rollingLogHistoryDelimiter = "." +) + +// Types of the rolling writer: roll by date, by time, etc. +type rollingType uint8 + +const ( + rollingTypeSize = iota + rollingTypeTime +) + +// Types of the rolled file naming mode: prefix, postfix, etc. +type rollingNameMode uint8 + +const ( + rollingNameModePostfix = iota + rollingNameModePrefix +) + +var rollingNameModesStringRepresentation = map[rollingNameMode]string{ + rollingNameModePostfix: "postfix", + rollingNameModePrefix: "prefix", +} + +func rollingNameModeFromString(rollingNameStr string) (rollingNameMode, bool) { + for tp, tpStr := range rollingNameModesStringRepresentation { + if tpStr == rollingNameStr { + return tp, true + } + } + + return 0, false +} + +type rollingIntervalType uint8 + +const ( + rollingIntervalAny = iota + rollingIntervalDaily +) + +var rollingInvervalTypesStringRepresentation = map[rollingIntervalType]string{ + rollingIntervalDaily: "daily", +} + +func rollingIntervalTypeFromString(rollingTypeStr string) (rollingIntervalType, bool) { + for tp, tpStr := range rollingInvervalTypesStringRepresentation { + if tpStr == rollingTypeStr { + return tp, true + } + } + + return 0, false +} + +var rollingTypesStringRepresentation = map[rollingType]string{ + rollingTypeSize: "size", + rollingTypeTime: "date", +} + +func rollingTypeFromString(rollingTypeStr string) (rollingType, bool) { + for tp, tpStr := range rollingTypesStringRepresentation { + if tpStr == rollingTypeStr { + return tp, true + } + } + + return 0, false +} + +// Old logs archivation type. +type rollingArchiveType uint8 + +const ( + rollingArchiveNone = iota + rollingArchiveZip +) + +var rollingArchiveTypesStringRepresentation = map[rollingArchiveType]string{ + rollingArchiveNone: "none", + rollingArchiveZip: "zip", +} + +func rollingArchiveTypeFromString(rollingArchiveTypeStr string) (rollingArchiveType, bool) { + for tp, tpStr := range rollingArchiveTypesStringRepresentation { + if tpStr == rollingArchiveTypeStr { + return tp, true + } + } + + return 0, false +} + +// Default names for different archivation types +var rollingArchiveTypesDefaultNames = map[rollingArchiveType]string{ + rollingArchiveZip: "log.zip", +} + +// rollerVirtual is an interface that represents all virtual funcs that are +// called in different rolling writer subtypes. +type rollerVirtual interface { + needsToRoll() (bool, error) // Returns true if needs to switch to another file. + isFileRollNameValid(rname string) bool // Returns true if logger roll file name (postfix/prefix/etc.) is ok. + sortFileRollNamesAsc(fs []string) ([]string, error) // Sorts logger roll file names in ascending order of their creation by logger. + + // Creates a new froll history file using the contents of current file and special filename of the latest roll (prefix/ postfix). + // If lastRollName is empty (""), then it means that there is no latest roll (current is the first one) + getNewHistoryRollFileName(lastRollName string) string + getCurrentModifiedFileName(originalFileName string, first bool) (string, error) // Returns filename modified according to specific logger rules +} + +// rollingFileWriter writes received messages to a file, until time interval passes +// or file exceeds a specified limit. After that the current log file is renamed +// and writer starts to log into a new file. You can set a limit for such renamed +// files count, if you want, and then the rolling writer would delete older ones when +// the files count exceed the specified limit. +type rollingFileWriter struct { + fileName string // current file name. May differ from original in date rolling loggers + originalFileName string // original one + currentDirPath string + currentFile *os.File + currentFileSize int64 + rollingType rollingType // Rolling mode (Files roll by size/date/...) + archiveType rollingArchiveType + archivePath string + maxRolls int + nameMode rollingNameMode + self rollerVirtual // Used for virtual calls +} + +func newRollingFileWriter(fpath string, rtype rollingType, atype rollingArchiveType, apath string, maxr int, namemode rollingNameMode) (*rollingFileWriter, error) { + rw := new(rollingFileWriter) + rw.currentDirPath, rw.fileName = filepath.Split(fpath) + if len(rw.currentDirPath) == 0 { + rw.currentDirPath = "." + } + rw.originalFileName = rw.fileName + + rw.rollingType = rtype + rw.archiveType = atype + rw.archivePath = apath + rw.nameMode = namemode + rw.maxRolls = maxr + return rw, nil +} + +func (rw *rollingFileWriter) hasRollName(file string) bool { + switch rw.nameMode { + case rollingNameModePostfix: + rname := rw.originalFileName + rollingLogHistoryDelimiter + return strings.HasPrefix(file, rname) + case rollingNameModePrefix: + rname := rollingLogHistoryDelimiter + rw.originalFileName + return strings.HasSuffix(file, rname) + } + return false +} + +func (rw *rollingFileWriter) createFullFileName(originalName, rollname string) string { + switch rw.nameMode { + case rollingNameModePostfix: + return originalName + rollingLogHistoryDelimiter + rollname + case rollingNameModePrefix: + return rollname + rollingLogHistoryDelimiter + originalName + } + return "" +} + +func (rw *rollingFileWriter) getSortedLogHistory() ([]string, error) { + files, err := getDirFilePaths(rw.currentDirPath, nil, true) + if err != nil { + return nil, err + } + var validRollNames []string + for _, file := range files { + if file != rw.fileName && rw.hasRollName(file) { + rname := rw.getFileRollName(file) + if rw.self.isFileRollNameValid(rname) { + validRollNames = append(validRollNames, rname) + } + } + } + sortedTails, err := rw.self.sortFileRollNamesAsc(validRollNames) + if err != nil { + return nil, err + } + validSortedFiles := make([]string, len(sortedTails)) + for i, v := range sortedTails { + validSortedFiles[i] = rw.createFullFileName(rw.originalFileName, v) + } + return validSortedFiles, nil +} + +func (rw *rollingFileWriter) createFileAndFolderIfNeeded(first bool) error { + var err error + + if len(rw.currentDirPath) != 0 { + err = os.MkdirAll(rw.currentDirPath, defaultDirectoryPermissions) + + if err != nil { + return err + } + } + + rw.fileName, err = rw.self.getCurrentModifiedFileName(rw.originalFileName, first) + if err != nil { + return err + } + filePath := filepath.Join(rw.currentDirPath, rw.fileName) + + // If exists + stat, err := os.Lstat(filePath) + if err == nil { + rw.currentFile, err = os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, defaultFilePermissions) + + stat, err = os.Lstat(filePath) + if err != nil { + return err + } + + rw.currentFileSize = stat.Size() + } else { + rw.currentFile, err = os.Create(filePath) + rw.currentFileSize = 0 + } + if err != nil { + return err + } + + return nil +} + +func (rw *rollingFileWriter) deleteOldRolls(history []string) error { + if rw.maxRolls <= 0 { + return nil + } + + rollsToDelete := len(history) - rw.maxRolls + if rollsToDelete <= 0 { + return nil + } + + switch rw.archiveType { + case rollingArchiveZip: + var files map[string][]byte + + // If archive exists + _, err := os.Lstat(rw.archivePath) + if nil == err { + // Extract files and content from it + files, err = unzip(rw.archivePath) + if err != nil { + return err + } + + // Remove the original file + err = tryRemoveFile(rw.archivePath) + if err != nil { + return err + } + } else { + files = make(map[string][]byte) + } + + // Add files to the existing files map, filled above + for i := 0; i < rollsToDelete; i++ { + rollPath := filepath.Join(rw.currentDirPath, history[i]) + bts, err := ioutil.ReadFile(rollPath) + if err != nil { + return err + } + + files[rollPath] = bts + } + + // Put the final file set to zip file. + if err = createZip(rw.archivePath, files); err != nil { + return err + } + } + var err error + // In all cases (archive files or not) the files should be deleted. + for i := 0; i < rollsToDelete; i++ { + // Try best to delete files without breaking the loop. + if err = tryRemoveFile(filepath.Join(rw.currentDirPath, history[i])); err != nil { + reportInternalError(err) + } + } + + return nil +} + +func (rw *rollingFileWriter) getFileRollName(fileName string) string { + switch rw.nameMode { + case rollingNameModePostfix: + return fileName[len(rw.originalFileName+rollingLogHistoryDelimiter):] + case rollingNameModePrefix: + return fileName[:len(fileName)-len(rw.originalFileName+rollingLogHistoryDelimiter)] + } + return "" +} + +func (rw *rollingFileWriter) Write(bytes []byte) (n int, err error) { + if rw.currentFile == nil { + err := rw.createFileAndFolderIfNeeded(true) + if err != nil { + return 0, err + } + } + // needs to roll if: + // * file roller max file size exceeded OR + // * time roller interval passed + nr, err := rw.self.needsToRoll() + if err != nil { + return 0, err + } + if nr { + // First, close current file. + err = rw.currentFile.Close() + if err != nil { + return 0, err + } + // Current history of all previous log files. + // For file roller it may be like this: + // * ... + // * file.log.4 + // * file.log.5 + // * file.log.6 + // + // For date roller it may look like this: + // * ... + // * file.log.11.Aug.13 + // * file.log.15.Aug.13 + // * file.log.16.Aug.13 + // Sorted log history does NOT include current file. + history, err := rw.getSortedLogHistory() + if err != nil { + return 0, err + } + // Renames current file to create a new roll history entry + // For file roller it may be like this: + // * ... + // * file.log.4 + // * file.log.5 + // * file.log.6 + // n file.log.7 <---- RENAMED (from file.log) + // Time rollers that doesn't modify file names (e.g. 'date' roller) skip this logic. + var newHistoryName string + var newRollMarkerName string + if len(history) > 0 { + // Create new rname name using last history file name + newRollMarkerName = rw.self.getNewHistoryRollFileName(rw.getFileRollName(history[len(history)-1])) + } else { + // Create first rname name + newRollMarkerName = rw.self.getNewHistoryRollFileName("") + } + if len(newRollMarkerName) != 0 { + newHistoryName = rw.createFullFileName(rw.fileName, newRollMarkerName) + } else { + newHistoryName = rw.fileName + } + if newHistoryName != rw.fileName { + err = os.Rename(filepath.Join(rw.currentDirPath, rw.fileName), filepath.Join(rw.currentDirPath, newHistoryName)) + if err != nil { + return 0, err + } + } + // Finally, add the newly added history file to the history archive + // and, if after that the archive exceeds the allowed max limit, older rolls + // must the removed/archived. + history = append(history, newHistoryName) + if len(history) > rw.maxRolls { + err = rw.deleteOldRolls(history) + if err != nil { + return 0, err + } + } + + err = rw.createFileAndFolderIfNeeded(false) + if err != nil { + return 0, err + } + } + + rw.currentFileSize += int64(len(bytes)) + return rw.currentFile.Write(bytes) +} + +func (rw *rollingFileWriter) Close() error { + if rw.currentFile != nil { + e := rw.currentFile.Close() + if e != nil { + return e + } + rw.currentFile = nil + } + return nil +} + +// ============================================================================================= +// Different types of rolling writers +// ============================================================================================= + +// -------------------------------------------------- +// Rolling writer by SIZE +// -------------------------------------------------- + +// rollingFileWriterSize performs roll when file exceeds a specified limit. +type rollingFileWriterSize struct { + *rollingFileWriter + maxFileSize int64 +} + +func NewRollingFileWriterSize(fpath string, atype rollingArchiveType, apath string, maxSize int64, maxRolls int, namemode rollingNameMode) (*rollingFileWriterSize, error) { + rw, err := newRollingFileWriter(fpath, rollingTypeSize, atype, apath, maxRolls, namemode) + if err != nil { + return nil, err + } + rws := &rollingFileWriterSize{rw, maxSize} + rws.self = rws + return rws, nil +} + +func (rws *rollingFileWriterSize) needsToRoll() (bool, error) { + return rws.currentFileSize >= rws.maxFileSize, nil +} + +func (rws *rollingFileWriterSize) isFileRollNameValid(rname string) bool { + if len(rname) == 0 { + return false + } + _, err := strconv.Atoi(rname) + return err == nil +} + +type rollSizeFileTailsSlice []string + +func (p rollSizeFileTailsSlice) Len() int { return len(p) } +func (p rollSizeFileTailsSlice) Less(i, j int) bool { + v1, _ := strconv.Atoi(p[i]) + v2, _ := strconv.Atoi(p[j]) + return v1 < v2 +} +func (p rollSizeFileTailsSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func (rws *rollingFileWriterSize) sortFileRollNamesAsc(fs []string) ([]string, error) { + ss := rollSizeFileTailsSlice(fs) + sort.Sort(ss) + return ss, nil +} + +func (rws *rollingFileWriterSize) getNewHistoryRollFileName(lastRollName string) string { + v := 0 + if len(lastRollName) != 0 { + v, _ = strconv.Atoi(lastRollName) + } + return fmt.Sprintf("%d", v+1) +} + +func (rws *rollingFileWriterSize) getCurrentModifiedFileName(originalFileName string, first bool) (string, error) { + return originalFileName, nil +} + +func (rws *rollingFileWriterSize) String() string { + return fmt.Sprintf("Rolling file writer (By SIZE): filename: %s, archive: %s, archivefile: %s, maxFileSize: %v, maxRolls: %v", + rws.fileName, + rollingArchiveTypesStringRepresentation[rws.archiveType], + rws.archivePath, + rws.maxFileSize, + rws.maxRolls) +} + +// -------------------------------------------------- +// Rolling writer by TIME +// -------------------------------------------------- + +// rollingFileWriterTime performs roll when a specified time interval has passed. +type rollingFileWriterTime struct { + *rollingFileWriter + timePattern string + interval rollingIntervalType + currentTimeFileName string +} + +func NewRollingFileWriterTime(fpath string, atype rollingArchiveType, apath string, maxr int, + timePattern string, interval rollingIntervalType, namemode rollingNameMode) (*rollingFileWriterTime, error) { + + rw, err := newRollingFileWriter(fpath, rollingTypeTime, atype, apath, maxr, namemode) + if err != nil { + return nil, err + } + rws := &rollingFileWriterTime{rw, timePattern, interval, ""} + rws.self = rws + return rws, nil +} + +func (rwt *rollingFileWriterTime) needsToRoll() (bool, error) { + switch rwt.nameMode { + case rollingNameModePostfix: + if rwt.originalFileName+rollingLogHistoryDelimiter+time.Now().Format(rwt.timePattern) == rwt.fileName { + return false, nil + } + case rollingNameModePrefix: + if time.Now().Format(rwt.timePattern)+rollingLogHistoryDelimiter+rwt.originalFileName == rwt.fileName { + return false, nil + } + } + if rwt.interval == rollingIntervalAny { + return true, nil + } + + tprev, err := time.ParseInLocation(rwt.timePattern, rwt.getFileRollName(rwt.fileName), time.Local) + if err != nil { + return false, err + } + + diff := time.Now().Sub(tprev) + switch rwt.interval { + case rollingIntervalDaily: + return diff >= 24*time.Hour, nil + } + return false, fmt.Errorf("unknown interval type: %d", rwt.interval) +} + +func (rwt *rollingFileWriterTime) isFileRollNameValid(rname string) bool { + if len(rname) == 0 { + return false + } + _, err := time.ParseInLocation(rwt.timePattern, rname, time.Local) + return err == nil +} + +type rollTimeFileTailsSlice struct { + data []string + pattern string +} + +func (p rollTimeFileTailsSlice) Len() int { return len(p.data) } + +func (p rollTimeFileTailsSlice) Less(i, j int) bool { + t1, _ := time.ParseInLocation(p.pattern, p.data[i], time.Local) + t2, _ := time.ParseInLocation(p.pattern, p.data[j], time.Local) + return t1.Before(t2) +} + +func (p rollTimeFileTailsSlice) Swap(i, j int) { p.data[i], p.data[j] = p.data[j], p.data[i] } + +func (rwt *rollingFileWriterTime) sortFileRollNamesAsc(fs []string) ([]string, error) { + ss := rollTimeFileTailsSlice{data: fs, pattern: rwt.timePattern} + sort.Sort(ss) + return ss.data, nil +} + +func (rwt *rollingFileWriterTime) getNewHistoryRollFileName(lastRollName string) string { + return "" +} + +func (rwt *rollingFileWriterTime) getCurrentModifiedFileName(originalFileName string, first bool) (string, error) { + if first { + history, err := rwt.getSortedLogHistory() + if err != nil { + return "", err + } + if len(history) > 0 { + return history[len(history)-1], nil + } + } + + switch rwt.nameMode { + case rollingNameModePostfix: + return originalFileName + rollingLogHistoryDelimiter + time.Now().Format(rwt.timePattern), nil + case rollingNameModePrefix: + return time.Now().Format(rwt.timePattern) + rollingLogHistoryDelimiter + originalFileName, nil + } + return "", fmt.Errorf("Unknown rolling writer mode. Either postfix or prefix must be used") +} + +func (rwt *rollingFileWriterTime) String() string { + return fmt.Sprintf("Rolling file writer (By TIME): filename: %s, archive: %s, archivefile: %s, maxInterval: %v, pattern: %s, maxRolls: %v", + rwt.fileName, + rollingArchiveTypesStringRepresentation[rwt.archiveType], + rwt.archivePath, + rwt.interval, + rwt.timePattern, + rwt.maxRolls) +} diff --git a/lib/seelog/writers_rollingfilewriter_test.go b/lib/seelog/writers_rollingfilewriter_test.go new file mode 100644 index 000000000..f929a3672 --- /dev/null +++ b/lib/seelog/writers_rollingfilewriter_test.go @@ -0,0 +1,99 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "fmt" + "io" + "testing" +) + +// fileWriterTestCase is declared in writers_filewriter_test.go + +func createRollingSizeFileWriterTestCase( + files []string, + fileName string, + fileSize int64, + maxRolls int, + writeCount int, + resFiles []string, + nameMode rollingNameMode) *fileWriterTestCase { + + return &fileWriterTestCase{files, fileName, rollingTypeSize, fileSize, maxRolls, "", writeCount, resFiles, nameMode} +} + +func createRollingDatefileWriterTestCase( + files []string, + fileName string, + datePattern string, + writeCount int, + resFiles []string, + nameMode rollingNameMode) *fileWriterTestCase { + + return &fileWriterTestCase{files, fileName, rollingTypeTime, 0, 0, datePattern, writeCount, resFiles, nameMode} +} + +func TestRollingFileWriter(t *testing.T) { + t.Logf("Starting rolling file writer tests") + NewFileWriterTester(rollingfileWriterTests, rollingFileWriterGetter, t).test() +} + +//=============================================================== + +func rollingFileWriterGetter(testCase *fileWriterTestCase) (io.WriteCloser, error) { + if testCase.rollingType == rollingTypeSize { + return NewRollingFileWriterSize(testCase.fileName, rollingArchiveNone, "", testCase.fileSize, testCase.maxRolls, testCase.nameMode) + } else if testCase.rollingType == rollingTypeTime { + return NewRollingFileWriterTime(testCase.fileName, rollingArchiveNone, "", -1, testCase.datePattern, rollingIntervalDaily, testCase.nameMode) + } + + return nil, fmt.Errorf("incorrect rollingType") +} + +// =============================================================== +var rollingfileWriterTests = []*fileWriterTestCase{ + createRollingSizeFileWriterTestCase([]string{}, "log.testlog", 10, 10, 1, []string{"log.testlog"}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{}, "log.testlog", 10, 10, 2, []string{"log.testlog", "log.testlog.1"}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{"1.log.testlog"}, "log.testlog", 10, 10, 2, []string{"log.testlog", "1.log.testlog", "2.log.testlog"}, rollingNameModePrefix), + createRollingSizeFileWriterTestCase([]string{"log.testlog.1"}, "log.testlog", 10, 1, 2, []string{"log.testlog", "log.testlog.2"}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{}, "log.testlog", 10, 1, 2, []string{"log.testlog", "log.testlog.1"}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{"log.testlog.9"}, "log.testlog", 10, 1, 2, []string{"log.testlog", "log.testlog.10"}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{"log.testlog.a", "log.testlog.1b"}, "log.testlog", 10, 1, 2, []string{"log.testlog", "log.testlog.1", "log.testlog.a", "log.testlog.1b"}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{}, `dir/log.testlog`, 10, 10, 1, []string{`dir/log.testlog`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{}, `dir/log.testlog`, 10, 10, 2, []string{`dir/log.testlog`, `dir/1.log.testlog`}, rollingNameModePrefix), + createRollingSizeFileWriterTestCase([]string{`dir/dir/log.testlog.1`}, `dir/dir/log.testlog`, 10, 10, 2, []string{`dir/dir/log.testlog`, `dir/dir/log.testlog.1`, `dir/dir/log.testlog.2`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{`dir/dir/dir/log.testlog.1`}, `dir/dir/dir/log.testlog`, 10, 1, 2, []string{`dir/dir/dir/log.testlog`, `dir/dir/dir/log.testlog.2`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{}, `./log.testlog`, 10, 1, 2, []string{`log.testlog`, `log.testlog.1`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{`././././log.testlog.9`}, `log.testlog`, 10, 1, 2, []string{`log.testlog`, `log.testlog.10`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{"dir/dir/log.testlog.a", "dir/dir/log.testlog.1b"}, "dir/dir/log.testlog", 10, 1, 2, []string{"dir/dir/log.testlog", "dir/dir/log.testlog.1", "dir/dir/log.testlog.a", "dir/dir/log.testlog.1b"}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{}, `././dir/log.testlog`, 10, 10, 1, []string{`dir/log.testlog`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{}, `././dir/log.testlog`, 10, 10, 2, []string{`dir/log.testlog`, `dir/log.testlog.1`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{`././dir/dir/log.testlog.1`}, `dir/dir/log.testlog`, 10, 10, 2, []string{`dir/dir/log.testlog`, `dir/dir/log.testlog.1`, `dir/dir/log.testlog.2`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{`././dir/dir/dir/log.testlog.1`}, `dir/dir/dir/log.testlog`, 10, 1, 2, []string{`dir/dir/dir/log.testlog`, `dir/dir/dir/log.testlog.2`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{}, `././log.testlog`, 10, 1, 2, []string{`log.testlog`, `log.testlog.1`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{`././././log.testlog.9`}, `log.testlog`, 10, 1, 2, []string{`log.testlog`, `log.testlog.10`}, rollingNameModePostfix), + createRollingSizeFileWriterTestCase([]string{"././dir/dir/log.testlog.a", "././dir/dir/log.testlog.1b"}, "dir/dir/log.testlog", 10, 1, 2, []string{"dir/dir/log.testlog", "dir/dir/log.testlog.1", "dir/dir/log.testlog.a", "dir/dir/log.testlog.1b"}, rollingNameModePostfix), + // ==================== +} diff --git a/lib/seelog/writers_smtpwriter.go b/lib/seelog/writers_smtpwriter.go new file mode 100644 index 000000000..31b794383 --- /dev/null +++ b/lib/seelog/writers_smtpwriter.go @@ -0,0 +1,214 @@ +// Copyright (c) 2012 - Cloud Instruments Co., Ltd. +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package seelog + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io/ioutil" + "net/smtp" + "path/filepath" + "strings" +) + +const ( + // Default subject phrase for sending emails. + DefaultSubjectPhrase = "Diagnostic message from server: " + + // Message subject pattern composed according to RFC 5321. + rfc5321SubjectPattern = "From: %s <%s>\nSubject: %s\n\n" +) + +// smtpWriter is used to send emails via given SMTP-server. +type smtpWriter struct { + auth smtp.Auth + hostName string + hostPort string + hostNameWithPort string + senderAddress string + senderName string + recipientAddresses []string + caCertDirPaths []string + mailHeaders []string + subject string +} + +// NewSMTPWriter returns a new SMTP-writer. +func NewSMTPWriter(sa, sn string, ras []string, hn, hp, un, pwd string, cacdps []string, subj string, headers []string) *smtpWriter { + return &smtpWriter{ + auth: smtp.PlainAuth("", un, pwd, hn), + hostName: hn, + hostPort: hp, + hostNameWithPort: fmt.Sprintf("%s:%s", hn, hp), + senderAddress: sa, + senderName: sn, + recipientAddresses: ras, + caCertDirPaths: cacdps, + subject: subj, + mailHeaders: headers, + } +} + +func prepareMessage(senderAddr, senderName, subject string, body []byte, headers []string) []byte { + headerLines := fmt.Sprintf(rfc5321SubjectPattern, senderName, senderAddr, subject) + // Build header lines if configured. + if headers != nil && len(headers) > 0 { + headerLines += strings.Join(headers, "\n") + headerLines += "\n" + } + return append([]byte(headerLines), body...) +} + +// getTLSConfig gets paths of PEM files with certificates, +// host server name and tries to create an appropriate TLS.Config. +func getTLSConfig(pemFileDirPaths []string, hostName string) (config *tls.Config, err error) { + if pemFileDirPaths == nil || len(pemFileDirPaths) == 0 { + err = errors.New("invalid PEM file paths") + return + } + pemEncodedContent := []byte{} + var ( + e error + bytes []byte + ) + // Create a file-filter-by-extension, set aside non-pem files. + pemFilePathFilter := func(fp string) bool { + if filepath.Ext(fp) == ".pem" { + return true + } + return false + } + for _, pemFileDirPath := range pemFileDirPaths { + pemFilePaths, err := getDirFilePaths(pemFileDirPath, pemFilePathFilter, false) + if err != nil { + return nil, err + } + + // Put together all the PEM files to decode them as a whole byte slice. + for _, pfp := range pemFilePaths { + if bytes, e = ioutil.ReadFile(pfp); e == nil { + pemEncodedContent = append(pemEncodedContent, bytes...) + } else { + return nil, fmt.Errorf("cannot read file: %s: %s", pfp, e.Error()) + } + } + } + config = &tls.Config{RootCAs: x509.NewCertPool(), ServerName: hostName} + isAppended := config.RootCAs.AppendCertsFromPEM(pemEncodedContent) + if !isAppended { + // Extract this into a separate error. + err = errors.New("invalid PEM content") + return + } + return +} + +// SendMail accepts TLS configuration, connects to the server at addr, +// switches to TLS if possible, authenticates with mechanism a if possible, +// and then sends an email from address from, to addresses to, with message msg. +func sendMailWithTLSConfig(config *tls.Config, addr string, a smtp.Auth, from string, to []string, msg []byte) error { + c, err := smtp.Dial(addr) + if err != nil { + return err + } + // Check if the server supports STARTTLS extension. + if ok, _ := c.Extension("STARTTLS"); ok { + if err = c.StartTLS(config); err != nil { + return err + } + } + // Check if the server supports AUTH extension and use given smtp.Auth. + if a != nil { + if isSupported, _ := c.Extension("AUTH"); isSupported { + if err = c.Auth(a); err != nil { + return err + } + } + } + // Portion of code from the official smtp.SendMail function, + // see http://golang.org/src/pkg/net/smtp/smtp.go. + if err = c.Mail(from); err != nil { + return err + } + for _, addr := range to { + if err = c.Rcpt(addr); err != nil { + return err + } + } + w, err := c.Data() + if err != nil { + return err + } + _, err = w.Write(msg) + if err != nil { + return err + } + err = w.Close() + if err != nil { + return err + } + return c.Quit() +} + +// Write pushes a text message properly composed according to RFC 5321 +// to a post server, which sends it to the recipients. +func (smtpw *smtpWriter) Write(data []byte) (int, error) { + var err error + + if smtpw.caCertDirPaths == nil { + err = smtp.SendMail( + smtpw.hostNameWithPort, + smtpw.auth, + smtpw.senderAddress, + smtpw.recipientAddresses, + prepareMessage(smtpw.senderAddress, smtpw.senderName, smtpw.subject, data, smtpw.mailHeaders), + ) + } else { + config, e := getTLSConfig(smtpw.caCertDirPaths, smtpw.hostName) + if e != nil { + return 0, e + } + err = sendMailWithTLSConfig( + config, + smtpw.hostNameWithPort, + smtpw.auth, + smtpw.senderAddress, + smtpw.recipientAddresses, + prepareMessage(smtpw.senderAddress, smtpw.senderName, smtpw.subject, data, smtpw.mailHeaders), + ) + } + if err != nil { + return 0, err + } + return len(data), nil +} + +// Close closes down SMTP-connection. +func (smtpw *smtpWriter) Close() error { + // Do nothing as Write method opens and closes connection automatically. + return nil +} diff --git a/lib/statsd/LICENSE b/lib/statsd/LICENSE new file mode 100644 index 000000000..569a9a3e3 --- /dev/null +++ b/lib/statsd/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Lorenzo Alberton + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/lib/statsd/README.md b/lib/statsd/README.md new file mode 100644 index 000000000..bc1d3622b --- /dev/null +++ b/lib/statsd/README.md @@ -0,0 +1,77 @@ +# StatsD client (Golang) + +[![GoDoc](https://godoc.org/github.com/quipo/statsd?status.png)](http://godoc.org/github.com/quipo/statsd) + +## Introduction + +Go Client library for [StatsD](https://github.com/etsy/statsd/). Contains a direct and a buffered client. +The buffered version will hold and aggregate values for the same key in memory before flushing them at the defined frequency. + +This client library was inspired by the one embedded in the [Bit.ly NSQ](https://github.com/bitly/nsq/blob/master/util/statsd_client.go) project, and extended to support some extra custom events used at DataSift. + +## Installation + + go get github.com/quipo/statsd + +## Supported event types + +* Increment - Count occurrences per second/minute of a specific event +* Decrement - Count occurrences per second/minute of a specific event +* Timing - To track a duration event +* Gauge - Gauges are a constant data type. They are not subject to averaging, and they don’t change unless you change them. That is, once you set a gauge value, it will be a flat line on the graph until you change it again +* Absolute - Absolute-valued metric (not averaged/aggregated) +* Total - Continously increasing value, e.g. read operations since boot + + +## Sample usage + +```go +package main + +import ( + "log" + "os" + "time" + + "github.com/quipo/statsd" +) + +func main() { + // init + prefix := "myproject." + statsdclient := statsd.NewStatsdClient("localhost:8125", prefix) + err := statsdclient.CreateSocket() + if nil != err { + log.Println(err) + os.Exit(1) + } + interval := time.Second * 2 // aggregate stats and flush every 2 seconds + stats := statsd.NewStatsdBuffer(interval, statsdclient) + defer stats.Close() + + // not buffered: send immediately + statsdclient.Incr("mymetric", 4) + + // buffered: aggregate in memory before flushing + stats.Incr("mymetric", 1) + stats.Incr("mymetric", 3) + stats.Incr("mymetric", 1) + stats.Incr("mymetric", 1) +} +``` + +The string "%HOST%" in the metric name will automatically be replaced with the hostname of the server the event is sent from. + + +## Author + +Lorenzo Alberton + +* Web: [http://alberton.info](http://alberton.info) +* Twitter: [@lorenzoalberton](https://twitter.com/lorenzoalberton) +* Linkedin: [/in/lorenzoalberton](https://www.linkedin.com/in/lorenzoalberton) + + +## Copyright + +See [LICENSE](LICENSE) document diff --git a/lib/statsd/bufferedclient.go b/lib/statsd/bufferedclient.go new file mode 100644 index 000000000..6ad3ea9c7 --- /dev/null +++ b/lib/statsd/bufferedclient.go @@ -0,0 +1,200 @@ +package statsd + +import ( + "time" + + log "github.com/cihub/seelog" + "infini.sh/framework/lib/statsd/event" +) + +// request to close the buffered statsd collector +type closeRequest struct { + reply chan error +} + +// StatsdBuffer is a client library to aggregate events in memory before +// flushing aggregates to StatsD, useful if the frequency of events is extremely high +// and sampling is not desirable +type StatsdBuffer struct { + statsd *StatsdClient + flushInterval time.Duration + eventChannel chan event.Event + events map[string]event.Event + closeChannel chan closeRequest + Verbose bool +} + +// NewStatsdBuffer Factory +func NewStatsdBuffer(interval time.Duration, buffSize int, client *StatsdClient) *StatsdBuffer { + sb := &StatsdBuffer{ + flushInterval: interval, + statsd: client, + eventChannel: make(chan event.Event, buffSize), + events: make(map[string]event.Event, 0), + closeChannel: make(chan closeRequest, 0), + Verbose: false, + } + go sb.collector() + return sb +} + +// CreateSocket creates a UDP connection to a StatsD server +func (sb *StatsdBuffer) CreateSocket() error { + return sb.statsd.CreateSocket() +} + +// CreateTCPSocket creates a TCP connection to a StatsD server +func (sb *StatsdBuffer) CreateTCPSocket() error { + return sb.statsd.CreateTCPSocket() +} + +// Incr - Increment a counter metric. Often used to note a particular event +func (sb *StatsdBuffer) Incr(stat string, count int64) error { + if 0 != count { + sb.eventChannel <- &event.Increment{Name: stat, Value: count} + } + return nil +} + +// Decr - Decrement a counter metric. Often used to note a particular event +func (sb *StatsdBuffer) Decr(stat string, count int64) error { + if 0 != count { + sb.eventChannel <- &event.Increment{Name: stat, Value: -count} + } + return nil +} + +// Timing - Track a duration event +func (sb *StatsdBuffer) Timing(stat string, delta int64) error { + sb.eventChannel <- event.NewTiming(stat, delta) + return nil +} + +// PrecisionTiming - Track a duration event +// the time delta has to be a duration +func (sb *StatsdBuffer) PrecisionTiming(stat string, delta time.Duration) error { + sb.eventChannel <- event.NewPrecisionTiming(stat, delta) + return nil +} + +// Gauge - Gauges are a constant data type. They are not subject to averaging, +// and they don’t change unless you change them. That is, once you set a gauge value, +// it will be a flat line on the graph until you change it again +func (sb *StatsdBuffer) Gauge(stat string, value int64) error { + sb.eventChannel <- &event.Gauge{Name: stat, Value: value} + return nil +} + +// GaugeDelta records a delta from the previous value (as int64) +func (sb *StatsdBuffer) GaugeDelta(stat string, value int64) error { + sb.eventChannel <- &event.GaugeDelta{Name: stat, Value: value} + return nil +} + +// FGauge is a Gauge working with float64 values +func (sb *StatsdBuffer) FGauge(stat string, value float64) error { + sb.eventChannel <- &event.FGauge{Name: stat, Value: value} + return nil +} + +// FGaugeDelta records a delta from the previous value (as float64) +func (sb *StatsdBuffer) FGaugeDelta(stat string, value float64) error { + sb.eventChannel <- &event.FGaugeDelta{Name: stat, Value: value} + return nil +} + +// Absolute - Send absolute-valued metric (not averaged/aggregated) +func (sb *StatsdBuffer) Absolute(stat string, value int64) error { + sb.eventChannel <- &event.Absolute{Name: stat, Values: []int64{value}} + return nil +} + +// FAbsolute - Send absolute-valued metric (not averaged/aggregated) +func (sb *StatsdBuffer) FAbsolute(stat string, value float64) error { + sb.eventChannel <- &event.FAbsolute{Name: stat, Values: []float64{value}} + return nil +} + +// Total - Send a metric that is continously increasing, e.g. read operations since boot +func (sb *StatsdBuffer) Total(stat string, value int64) error { + sb.eventChannel <- &event.Total{Name: stat, Value: value} + return nil +} + +// handle flushes and updates in one single thread (instead of locking the events map) +func (sb *StatsdBuffer) collector() { + // on a panic event, flush all the pending stats before panicking + defer func(sb *StatsdBuffer) { + if r := recover(); r != nil { + log.Error("Caught panic, flushing stats before throwing the panic again") + sb.flush() + panic(r) + } + }(sb) + + ticker := time.NewTicker(sb.flushInterval) + + for { + select { + case <-ticker.C: + //sb.Logger.Println("Flushing stats") + sb.flush() + case e := <-sb.eventChannel: + //sb.Logger.Println("Received ", e.String()) + // issue #28: unable to use Incr and PrecisionTiming with the same key (also fixed #27) + k := e.TypeString() + "|" + e.Key() + if e2, ok := sb.events[k]; ok { + //sb.Logger.Println("Updating existing event") + e2.Update(e) + sb.events[k] = e2 + } else { + //sb.Logger.Println("Adding new event") + sb.events[k] = e + } + case c := <-sb.closeChannel: + if sb.Verbose { + log.Error("Asked to terminate. Flushing stats before returning.") + } + c.reply <- sb.flush() + return + } + } +} + +// Close sends a close event to the collector asking to stop & flush pending stats +// and closes the statsd client +func (sb *StatsdBuffer) Close() (err error) { + // 1. send a close event to the collector + req := closeRequest{reply: make(chan error, 0)} + sb.closeChannel <- req + // 2. wait for the collector to drain the queue and respond + err = <-req.reply + // 3. close the statsd client + err2 := sb.statsd.Close() + if err != nil { + return err + } + return err2 +} + +// send the events to StatsD and reset them. +// This function is NOT thread-safe, so it must only be invoked synchronously +// from within the collector() goroutine +func (sb *StatsdBuffer) flush() (err error) { + n := len(sb.events) + if n == 0 { + return nil + } + err = sb.statsd.CreateSocket() + if nil != err { + log.Error("Error establishing UDP connection for sending statsd events:", err) + return err + } + if err := sb.statsd.SendEvents(sb.events); err != nil { + log.Error(err) + return err + } + sb.events = make(map[string]event.Event) + + return nil +} diff --git a/lib/statsd/bufferedclient_test.go b/lib/statsd/bufferedclient_test.go new file mode 100644 index 000000000..7f4b51019 --- /dev/null +++ b/lib/statsd/bufferedclient_test.go @@ -0,0 +1,86 @@ +package statsd + +import ( + "fmt" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + "time" +) + +func TestBufferedTotal(t *testing.T) { + ln, udpAddr := newLocalListenerUDP(t) + defer ln.Close() + + prefix := "myproject." + + client := NewStatsdClient(udpAddr.String(), prefix) + buffered := NewStatsdBuffer(time.Millisecond*20, 100, client) + + ch := make(chan string, 0) + + s := map[string]int64{ + "a:b:c": 5, + "d:e:f": 2, + "x:b:c": 5, + "g.h.i": 1, + } + + expected := make(map[string]int64) + for k, v := range s { + expected[k] = v + } + + // also test %HOST% replacement + s["zz.%HOST%"] = 1 + hostname, err := os.Hostname() + expected["zz."+hostname] = 1 + + go doListenUDP(t, ln, ch, len(s)) + + err = buffered.CreateSocket() + if nil != err { + t.Fatal(err) + } + defer buffered.Close() + + for k, v := range s { + buffered.Total(k, v) + } + + actual := make(map[string]int64) + + re := regexp.MustCompile(`^(.*)\:(\d+)\|(\w).*$`) + + received := 0 + + for received < len(s) { + batch := <-ch + for _, x := range strings.Split(batch, "\n") { + x = strings.TrimSpace(x) + //fmt.Println(x) + if !strings.HasPrefix(x, prefix) { + t.Errorf("Metric without expected prefix: expected '%s', actual '%s'", prefix, x) + return + } + received++ + vv := re.FindStringSubmatch(x) + fmt.Println(vv, x) + if vv[3] != "t" { + t.Errorf("Metric without expected suffix: expected 't', actual '%s'", vv[3]) + } + v, err := strconv.ParseInt(vv[2], 10, 64) + if err != nil { + t.Error(err) + } + actual[vv[1][len(prefix):]] = v + } + } + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("did not receive all metrics: Expected: %T %v, Actual: %T %v ", expected, expected, actual, actual) + } +} diff --git a/lib/statsd/client.go b/lib/statsd/client.go new file mode 100644 index 000000000..c9ff4fffc --- /dev/null +++ b/lib/statsd/client.go @@ -0,0 +1,257 @@ +package statsd + +import ( + "fmt" + "log" + "net" + "os" + "strings" + "sync" + "time" + + "infini.sh/framework/lib/statsd/event" +) + +// Logger interface compatible with log.Logger +type Logger interface { + Println(v ...interface{}) +} + +// UDPPayloadSize is the number of bytes to send at one go through the udp socket. +// SendEvents will try to pack as many events into one udp packet. +// Change this value as per network capabilities +// For example to change to 16KB +// +// import "github.com/quipo/statsd" +// func init() { +// statsd.UDPPayloadSize = 16 * 1024 +// } +var UDPPayloadSize int = 512 + +// Hostname is exported so clients can set it to something different than the default +var Hostname string + +var errNotConnected = fmt.Errorf("cannot send stats, not connected to StatsD server") + +func init() { + host, err := os.Hostname() + if nil == err { + Hostname = host + } +} + +// StatsdClient is a client library to send events to StatsD +type StatsdClient struct { + conn net.Conn + addr string + prefix string + eventStringTpl string + Logger Logger + lock sync.RWMutex +} + +// NewStatsdClient - Factory +func NewStatsdClient(addr string, prefix string) *StatsdClient { + // allow %HOST% in the prefix string + prefix = strings.Replace(prefix, "%HOST%", Hostname, 1) + return &StatsdClient{ + addr: addr, + prefix: prefix, + Logger: log.New(os.Stdout, "[StatsdClient] ", log.Ldate|log.Ltime), + eventStringTpl: "%s%s:%s", + } +} + +// String returns the StatsD server address +func (c *StatsdClient) String() string { + return c.addr +} + +// CreateSocket creates a UDP connection to a StatsD server +func (c *StatsdClient) CreateSocket() error { + conn, err := net.DialTimeout("udp", c.addr, 5*time.Second) + if err != nil { + return err + } + c.lock.Lock() + c.conn = conn + c.lock.Unlock() + return nil +} + +// CreateTCPSocket creates a TCP connection to a StatsD server +func (c *StatsdClient) CreateTCPSocket() error { + conn, err := net.DialTimeout("tcp", c.addr, 5*time.Second) + if err != nil { + return err + } + c.conn = conn + c.eventStringTpl = "%s%s:%s\n" + return nil +} + +// Close the UDP connection +func (c *StatsdClient) Close() error { + c.lock.Lock() + defer c.lock.Unlock() + + if nil == c.conn { + return nil + } + return c.conn.Close() +} + +// See statsd data types here: http://statsd.readthedocs.org/en/latest/types.html +// or also https://github.com/b/statsd_spec + +// Incr - Increment a counter metric. Often used to note a particular event +func (c *StatsdClient) Incr(stat string, count int64) error { + if 0 != count { + return c.send(stat, "%d|c", count) + } + return nil +} + +// Decr - Decrement a counter metric. Often used to note a particular event +func (c *StatsdClient) Decr(stat string, count int64) error { + if 0 != count { + return c.send(stat, "%d|c", -count) + } + return nil +} + +// Timing - Track a duration event +// the time delta must be given in milliseconds +func (c *StatsdClient) Timing(stat string, delta int64) error { + return c.send(stat, "%d|ms", delta) +} + +// PrecisionTiming - Track a duration event +// the time delta has to be a duration +func (c *StatsdClient) PrecisionTiming(stat string, delta time.Duration) error { + return c.send(stat, "%.6f|ms", float64(delta)/float64(time.Millisecond)) +} + +// Gauge - Gauges are a constant data type. They are not subject to averaging, +// and they don’t change unless you change them. That is, once you set a gauge value, +// it will be a flat line on the graph until you change it again. If you specify +// delta to be true, that specifies that the gauge should be updated, not set. Due to the +// underlying protocol, you can't explicitly set a gauge to a negative number without +// first setting it to zero. +func (c *StatsdClient) Gauge(stat string, value int64) error { + if value < 0 { + c.send(stat, "%d|g", 0) + return c.send(stat, "%d|g", value) + } + return c.send(stat, "%d|g", value) +} + +// GaugeDelta -- Send a change for a gauge +func (c *StatsdClient) GaugeDelta(stat string, value int64) error { + // Gauge Deltas are always sent with a leading '+' or '-'. The '-' takes care of itself but the '+' must added by hand + if value < 0 { + return c.send(stat, "%d|g", value) + } + return c.send(stat, "+%d|g", value) +} + +// FGauge -- Send a floating point value for a gauge +func (c *StatsdClient) FGauge(stat string, value float64) error { + if value < 0 { + c.send(stat, "%d|g", 0) + return c.send(stat, "%g|g", value) + } + return c.send(stat, "%g|g", value) +} + +// FGaugeDelta -- Send a floating point change for a gauge +func (c *StatsdClient) FGaugeDelta(stat string, value float64) error { + if value < 0 { + return c.send(stat, "%g|g", value) + } + return c.send(stat, "+%g|g", value) +} + +// Absolute - Send absolute-valued metric (not averaged/aggregated) +func (c *StatsdClient) Absolute(stat string, value int64) error { + return c.send(stat, "%d|a", value) +} + +// FAbsolute - Send absolute-valued floating point metric (not averaged/aggregated) +func (c *StatsdClient) FAbsolute(stat string, value float64) error { + return c.send(stat, "%g|a", value) +} + +// Total - Send a metric that is continously increasing, e.g. read operations since boot +func (c *StatsdClient) Total(stat string, value int64) error { + return c.send(stat, "%d|t", value) +} + +// write a UDP packet with the statsd event +func (c *StatsdClient) send(stat string, format string, value interface{}) error { + if c.conn == nil { + return errNotConnected + } + stat = strings.Replace(stat, "%HOST%", Hostname, 1) + // if sending tcp append a newline + format = fmt.Sprintf(c.eventStringTpl, c.prefix, stat, format) + _, err := fmt.Fprintf(c.conn, format, value) + return err +} + +// SendEvent - Sends stats from an event object +func (c *StatsdClient) SendEvent(e event.Event) error { + if c.conn == nil { + return errNotConnected + } + for _, stat := range e.Stats() { + //fmt.Printf("SENDING EVENT %s%s\n", c.prefix, strings.Replace(stat, "%HOST%", Hostname, 1)) + _, err := fmt.Fprintf(c.conn, "%s%s", c.prefix, strings.Replace(stat, "%HOST%", Hostname, 1)) + if nil != err { + return err + } + } + return nil +} + +// SendEvents - Sends stats from all the event objects. +// Tries to bundle many together into one fmt.Fprintf based on UDPPayloadSize. +func (c *StatsdClient) SendEvents(events map[string]event.Event) error { + if c.conn == nil { + return errNotConnected + } + + var n int + var stats []string = make([]string, 0) + + for _, e := range events { + for _, stat := range e.Stats() { + + stat = fmt.Sprintf("%s%s", c.prefix, strings.Replace(stat, "%HOST%", Hostname, 1)) + _n := n + len(stat) + 1 + + if _n > UDPPayloadSize { + // with this last event, the UDP payload would be too big + if _, err := fmt.Fprint(c.conn, strings.Join(stats, "\n")); err != nil { + return err + } + // reset payload after flushing, and add the last event + stats = []string{stat} + n = len(stat) + continue + } + + // can fit more into the current payload + n = _n + stats = append(stats, stat) + } + } + + if len(stats) != 0 { + if _, err := fmt.Fprint(c.conn, strings.Join(stats, "\n")); err != nil { + return err + } + } + + return nil +} diff --git a/lib/statsd/client_test.go b/lib/statsd/client_test.go new file mode 100644 index 000000000..83df04bf7 --- /dev/null +++ b/lib/statsd/client_test.go @@ -0,0 +1,315 @@ +package statsd + +import ( + "bytes" + "errors" + "fmt" + "net" + "os" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "infini.sh/framework/lib/statsd/event" +) + +// MockNetConn is a mock for net.Conn +type MockNetConn struct { + buf bytes.Buffer +} + +func (mock *MockNetConn) Read(b []byte) (n int, err error) { + return mock.buf.Read(b) +} +func (mock *MockNetConn) Write(b []byte) (n int, err error) { + return mock.buf.Write(append(b, '\n')) +} +func (mock MockNetConn) Close() error { + mock.buf.Truncate(0) + return nil +} +func (mock MockNetConn) LocalAddr() net.Addr { + return nil +} +func (mock MockNetConn) RemoteAddr() net.Addr { + return nil +} +func (mock MockNetConn) SetDeadline(t time.Time) error { + return nil +} +func (mock MockNetConn) SetReadDeadline(t time.Time) error { + return nil +} +func (mock MockNetConn) SetWriteDeadline(t time.Time) error { + return nil +} + +func newLocalListenerUDP(t *testing.T) (*net.UDPConn, *net.UDPAddr) { + addr := fmt.Sprintf(":%d", getFreePort()) + udpAddr, err := net.ResolveUDPAddr("udp", addr) + if err != nil { + t.Fatal(err) + } + ln, err := net.ListenUDP("udp", udpAddr) + if err != nil { + t.Fatal(err) + } + return ln, udpAddr +} + +func TestTotal(t *testing.T) { + ln, udpAddr := newLocalListenerUDP(t) + defer ln.Close() + + prefix := "myproject." + + client := NewStatsdClient(udpAddr.String(), prefix) + + ch := make(chan string, 0) + + s := map[string]int64{ + "a:b:c": 5, + "d:e:f": 2, + "x:b:c": 5, + "g.h.i": 1, + } + + expected := make(map[string]int64) + for k, v := range s { + expected[k] = v + } + + // also test %HOST% replacement + s["zz.%HOST%"] = 1 + hostname, err := os.Hostname() + expected["zz."+hostname] = 1 + + go doListenUDP(t, ln, ch, len(s)) + + err = client.CreateSocket() + if nil != err { + t.Fatal(err) + } + defer client.Close() + + for k, v := range s { + client.Total(k, v) + } + + actual := make(map[string]int64) + + re := regexp.MustCompile(`^(.*)\:(\d+)\|(\w).*$`) + + for i := len(s); i > 0; i-- { + x := <-ch + x = strings.TrimSpace(x) + //fmt.Println(x) + if !strings.HasPrefix(x, prefix) { + t.Errorf("Metric without expected prefix: expected '%s', actual '%s'", prefix, x) + break + } + vv := re.FindStringSubmatch(x) + if vv[3] != "t" { + t.Errorf("Metric without expected suffix: expected 't', actual '%s'", vv[3]) + } + v, err := strconv.ParseInt(vv[2], 10, 64) + if err != nil { + t.Error(err) + } + actual[vv[1][len(prefix):]] = v + } + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("did not receive all metrics: Expected: %T %v, Actual: %T %v ", expected, expected, actual, actual) + } +} + +func doListenUDP(t *testing.T, conn *net.UDPConn, ch chan string, n int) { + for n > 0 { + buffer := make([]byte, 1024) + size, err := conn.Read(buffer) + if err != nil { + if isClosedConnErr(err) { + return + } + t.Errorf("udp read failed: %v", err) + return + } + ch <- string(buffer[:size]) + n-- + } +} + +func doListenTCP(t *testing.T, conn net.Listener, ch chan string, n int) { + client, err := conn.Accept() + if err != nil { + if isClosedConnErr(err) { + return + } + t.Errorf("tcp accept failed: %v", err) + return + } + defer client.Close() + + for n > 0 { + buf := make([]byte, 1024) + c, err := client.Read(buf) + if err != nil { + if isClosedConnErr(err) { + return + } + t.Errorf("tcp read failed: %v", err) + return + } + + for _, s := range bytes.Split(buf[:c], []byte{'\n'}) { + if len(s) == 0 { + continue + } + ch <- string(s) + n-- + if n == 0 { + return + } + } + } +} + +func isClosedConnErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, net.ErrClosed) { + return true + } + // Some platforms can return the legacy text form without wrapping net.ErrClosed. + return strings.Contains(err.Error(), "use of closed network connection") +} + +func newLocalListenerTCP(t *testing.T) (string, net.Listener) { + addr := fmt.Sprintf("127.0.0.1:%d", getFreePort()) + ln, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + return addr, ln +} + +func TestTCP(t *testing.T) { + addr, ln := newLocalListenerTCP(t) + defer ln.Close() + + prefix := "myproject." + client := NewStatsdClient(addr, prefix) + + ch := make(chan string, 0) + + s := map[string]int64{ + "a:b:c": 5, + "d:e:f": 2, + "x:b:c": 5, + "g.h.i": 1, + } + + expected := make(map[string]int64) + for k, v := range s { + expected[k] = v + } + + // also test %HOST% replacement + s["zz.%HOST%"] = 1 + hostname, err := os.Hostname() + expected["zz."+hostname] = 1 + + go doListenTCP(t, ln, ch, len(s)) + + err = client.CreateTCPSocket() + if nil != err { + t.Fatal(err) + } + defer client.Close() + + for k, v := range s { + client.Total(k, v) + } + + actual := make(map[string]int64) + + re := regexp.MustCompile(`^(.*)\:(\d+)\|(\w).*$`) + + for i := len(s); i > 0; i-- { + x := <-ch + x = strings.TrimSpace(x) + //fmt.Println(x) + if !strings.HasPrefix(x, prefix) { + t.Errorf("Metric without expected prefix: expected '%s', actual '%s'", prefix, x) + break + } + vv := re.FindStringSubmatch(x) + if vv[3] != "t" { + t.Errorf("Metric without expected suffix: expected 't', actual '%s'", vv[3]) + } + v, err := strconv.ParseInt(vv[2], 10, 64) + if err != nil { + t.Error(err) + } + actual[vv[1][len(prefix):]] = v + } + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("did not receive all metrics: Expected: %T %v, Actual: %T %v \n", expected, expected, actual, actual) + } +} + +func TestSendEvents(t *testing.T) { + c := NewStatsdClient("127.0.0.1:1201", "test") + c.conn = &MockNetConn{} // mock connection + + // override with a small size + UDPPayloadSize = 40 + + e1 := &event.Increment{Name: "test1", Value: 123} + e2 := &event.Increment{Name: "test2", Value: 432} + e3 := &event.Increment{Name: "test3", Value: 111} + e4 := &event.Gauge{Name: "test4", Value: 12435} + + events := map[string]event.Event{ + "test1": e1, + "test2": e2, + "test3": e3, + "test4": e4, + } + + err := c.SendEvents(events) + if nil != err { + t.Error(err) + } + + b1 := make([]byte, UDPPayloadSize*3) + n, err2 := c.conn.Read(b1) + if nil != err2 { + t.Error(err2) + } + nStats := len(strings.Split(strings.TrimSpace(string(b1[:n])), "\n")) + if nStats != len(events) { + t.Errorf("Was expecting %d events, got %d: %s", len(events), nStats, string(b1)) + } +} + +// getFreePort Ask the kernel for a free open port that is ready to use +func getFreePort() int { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + panic(err) + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + panic(err) + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port +} diff --git a/lib/statsd/event/absolute.go b/lib/statsd/event/absolute.go new file mode 100644 index 000000000..71ae3adc1 --- /dev/null +++ b/lib/statsd/event/absolute.go @@ -0,0 +1,58 @@ +package event + +import "fmt" + +// Absolute is a metric that is not averaged/aggregated. +// We keep each value distinct and then we flush them all individually. +type Absolute struct { + Name string + Values []int64 +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *Absolute) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + e.Values = append(e.Values, e2.Payload().([]int64)...) + return nil +} + +// Payload returns the aggregated value for this event +func (e Absolute) Payload() interface{} { + return e.Values +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e Absolute) Stats() []string { + ret := make([]string, 0, len(e.Values)) + for _, v := range e.Values { + ret = append(ret, fmt.Sprintf("%s:%d|a", e.Name, v)) + } + return ret +} + +// Key returns the name of this metric +func (e Absolute) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *Absolute) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e Absolute) Type() int { + return EventAbsolute +} + +// TypeString returns a name for this type of metric +func (e Absolute) TypeString() string { + return "Absolute" +} + +// String returns a debug-friendly representation of this metric +func (e Absolute) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Values: %v}", e.TypeString(), e.Name, e.Values) +} diff --git a/lib/statsd/event/fabsolute.go b/lib/statsd/event/fabsolute.go new file mode 100644 index 000000000..c92d35832 --- /dev/null +++ b/lib/statsd/event/fabsolute.go @@ -0,0 +1,58 @@ +package event + +import "fmt" + +// FAbsolute is a metric that is not averaged/aggregated. +// We keep each value distinct and then we flush them all individually. +type FAbsolute struct { + Name string + Values []float64 +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *FAbsolute) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + e.Values = append(e.Values, e2.Payload().([]float64)...) + return nil +} + +// Payload returns the aggregated value for this event +func (e FAbsolute) Payload() interface{} { + return e.Values +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e FAbsolute) Stats() []string { + ret := make([]string, 0, len(e.Values)) + for _, v := range e.Values { + ret = append(ret, fmt.Sprintf("%s:%g|a", e.Name, v)) + } + return ret +} + +// Key returns the name of this metric +func (e FAbsolute) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *FAbsolute) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e FAbsolute) Type() int { + return EventFAbsolute +} + +// TypeString returns a name for this type of metric +func (e FAbsolute) TypeString() string { + return "FAbsolute" +} + +// String returns a debug-friendly representation of this metric +func (e FAbsolute) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Values: %v}", e.TypeString(), e.Name, e.Values) +} diff --git a/lib/statsd/event/fgauge.go b/lib/statsd/event/fgauge.go new file mode 100644 index 000000000..1b7cbeafd --- /dev/null +++ b/lib/statsd/event/fgauge.go @@ -0,0 +1,64 @@ +package event + +import "fmt" + +// FGauge - Gauges are a constant data type. They are not subject to averaging, +// and they don’t change unless you change them. That is, once you set a gauge value, +// it will be a flat line on the graph until you change it again +type FGauge struct { + Name string + Value float64 +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *FGauge) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + e.Value += e2.Payload().(float64) + return nil +} + +// Payload returns the aggregated value for this event +func (e FGauge) Payload() interface{} { + return e.Value +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e FGauge) Stats() []string { + if e.Value < 0 { + // because a leading '+' or '-' in the value of a gauge denotes a delta, to send + // a negative gauge value we first set the gauge absolutely to 0, then send the + // negative value as a delta from 0 (that's just how the spec works :-) + return []string{ + fmt.Sprintf("%s:%d|g", e.Name, 0), + fmt.Sprintf("%s:%g|g", e.Name, e.Value), + } + } + return []string{fmt.Sprintf("%s:%g|g", e.Name, e.Value)} +} + +// Key returns the name of this metric +func (e FGauge) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *FGauge) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e FGauge) Type() int { + return EventFGauge +} + +// TypeString returns a name for this type of metric +func (e FGauge) TypeString() string { + return "FGauge" +} + +// String returns a debug-friendly representation of this metric +func (e FGauge) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Value: %g}", e.TypeString(), e.Name, e.Value) +} diff --git a/lib/statsd/event/fgaugedelta.go b/lib/statsd/event/fgaugedelta.go new file mode 100644 index 000000000..450af2f33 --- /dev/null +++ b/lib/statsd/event/fgaugedelta.go @@ -0,0 +1,64 @@ +package event + +import "fmt" + +// FGaugeDelta - Gauges are a constant data type. They are not subject to averaging, +// and they don’t change unless you change them. That is, once you set a gauge value, +// it will be a flat line on the graph until you change it again +type FGaugeDelta struct { + Name string + Value float64 +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *FGaugeDelta) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + e.Value += e2.Payload().(float64) + return nil +} + +// Payload returns the aggregated value for this event +func (e FGaugeDelta) Payload() interface{} { + return e.Value +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e FGaugeDelta) Stats() []string { + if e.Value < 0 { + // because a leading '+' or '-' in the value of a gauge denotes a delta, to send + // a negative gauge value we first set the gauge absolutely to 0, then send the + // negative value as a delta from 0 (that's just how the spec works :-) + return []string{ + fmt.Sprintf("%s:%d|g", e.Name, 0), + fmt.Sprintf("%s:%g|g", e.Name, e.Value), + } + } + return []string{fmt.Sprintf("%s:%g|g", e.Name, e.Value)} +} + +// Key returns the name of this metric +func (e FGaugeDelta) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *FGaugeDelta) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e FGaugeDelta) Type() int { + return EventFGaugeDelta +} + +// TypeString returns a name for this type of metric +func (e FGaugeDelta) TypeString() string { + return "FGaugeDelta" +} + +// String returns a debug-friendly representation of this metric +func (e FGaugeDelta) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Value: %g}", e.TypeString(), e.Name, e.Value) +} diff --git a/lib/statsd/event/gauge.go b/lib/statsd/event/gauge.go new file mode 100644 index 000000000..f303536f7 --- /dev/null +++ b/lib/statsd/event/gauge.go @@ -0,0 +1,64 @@ +package event + +import "fmt" + +// Gauge - Gauges are a constant data type. They are not subject to averaging, +// and they don’t change unless you change them. That is, once you set a gauge value, +// it will be a flat line on the graph until you change it again +type Gauge struct { + Name string + Value int64 +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *Gauge) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + e.Value = e2.Payload().(int64) + return nil +} + +// Payload returns the aggregated value for this event +func (e Gauge) Payload() interface{} { + return e.Value +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e Gauge) Stats() []string { + if e.Value < 0 { + // because a leading '+' or '-' in the value of a gauge denotes a delta, to send + // a negative gauge value we first set the gauge absolutely to 0, then send the + // negative value as a delta from 0 (that's just how the spec works :-) + return []string{ + fmt.Sprintf("%s:%d|g", e.Name, 0), + fmt.Sprintf("%s:%d|g", e.Name, e.Value), + } + } + return []string{fmt.Sprintf("%s:%d|g", e.Name, e.Value)} +} + +// Key returns the name of this metric +func (e Gauge) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *Gauge) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e Gauge) Type() int { + return EventGauge +} + +// TypeString returns a name for this type of metric +func (e Gauge) TypeString() string { + return "Gauge" +} + +// String returns a debug-friendly representation of this metric +func (e Gauge) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Value: %d}", e.TypeString(), e.Name, e.Value) +} diff --git a/lib/statsd/event/gaugedelta.go b/lib/statsd/event/gaugedelta.go new file mode 100644 index 000000000..f2885e366 --- /dev/null +++ b/lib/statsd/event/gaugedelta.go @@ -0,0 +1,64 @@ +package event + +import "fmt" + +// GaugeDelta - Gauges are a constant data type. They are not subject to averaging, +// and they don’t change unless you change them. That is, once you set a gauge value, +// it will be a flat line on the graph until you change it again +type GaugeDelta struct { + Name string + Value int64 +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *GaugeDelta) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + e.Value += e2.Payload().(int64) + return nil +} + +// Payload returns the aggregated value for this event +func (e GaugeDelta) Payload() interface{} { + return e.Value +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e GaugeDelta) Stats() []string { + if e.Value < 0 { + // because a leading '+' or '-' in the value of a gauge denotes a delta, to send + // a negative gauge value we first set the gauge absolutely to 0, then send the + // negative value as a delta from 0 (that's just how the spec works :-) + return []string{ + fmt.Sprintf("%s:%d|g", e.Name, 0), + fmt.Sprintf("%s:%d|g", e.Name, e.Value), + } + } + return []string{fmt.Sprintf("%s:+%d|g", e.Name, e.Value)} +} + +// Key returns the name of this metric +func (e GaugeDelta) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *GaugeDelta) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e GaugeDelta) Type() int { + return EventGaugeDelta +} + +// TypeString returns a name for this type of metric +func (e GaugeDelta) TypeString() string { + return "GaugeDelta" +} + +// String returns a debug-friendly representation of this metric +func (e GaugeDelta) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Value: %d}", e.TypeString(), e.Name, e.Value) +} diff --git a/lib/statsd/event/increment.go b/lib/statsd/event/increment.go new file mode 100644 index 000000000..e61708389 --- /dev/null +++ b/lib/statsd/event/increment.go @@ -0,0 +1,53 @@ +package event + +import "fmt" + +// Increment represents a metric whose value is averaged over a minute +type Increment struct { + Name string + Value int64 +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *Increment) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + e.Value += e2.Payload().(int64) + return nil +} + +// Payload returns the aggregated value for this event +func (e Increment) Payload() interface{} { + return e.Value +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e Increment) Stats() []string { + return []string{fmt.Sprintf("%s:%d|c", e.Name, e.Value)} +} + +// Key returns the name of this metric +func (e Increment) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *Increment) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e Increment) Type() int { + return EventIncr +} + +// TypeString returns a name for this type of metric +func (e Increment) TypeString() string { + return "Increment" +} + +// String returns a debug-friendly representation of this metric +func (e Increment) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Value: %d}", e.TypeString(), e.Name, e.Value) +} diff --git a/lib/statsd/event/interface.go b/lib/statsd/event/interface.go new file mode 100644 index 000000000..f1748550e --- /dev/null +++ b/lib/statsd/event/interface.go @@ -0,0 +1,27 @@ +package event + +// constant event type identifiers +const ( + EventIncr = iota + EventTiming + EventAbsolute + EventTotal + EventGauge + EventGaugeDelta + EventFGauge + EventFGaugeDelta + EventFAbsolute + EventPrecisionTiming +) + +// Event is an interface to a generic StatsD event, used by the buffered client collator +type Event interface { + Stats() []string + Type() int + TypeString() string + Payload() interface{} + Update(e2 Event) error + String() string + Key() string + SetKey(string) +} diff --git a/lib/statsd/event/precisiontiming.go b/lib/statsd/event/precisiontiming.go new file mode 100644 index 000000000..3fef07536 --- /dev/null +++ b/lib/statsd/event/precisiontiming.go @@ -0,0 +1,78 @@ +package event + +import ( + "fmt" + "time" +) + +// PrecisionTiming keeps min/max/avg information about a timer over a certain interval +type PrecisionTiming struct { + Name string + Min time.Duration + Max time.Duration + Value time.Duration + Count int64 +} + +// NewPrecisionTiming is a factory for a Timing event, setting the Count to 1 to prevent div_by_0 errors +func NewPrecisionTiming(k string, delta time.Duration) *PrecisionTiming { + return &PrecisionTiming{Name: k, Min: delta, Max: delta, Value: delta, Count: 1} +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *PrecisionTiming) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + p := e2.Payload().(PrecisionTiming) + e.Count += p.Count + e.Value += p.Value + e.Min = time.Duration(minInt64(int64(e.Min), int64(p.Min))) + e.Max = time.Duration(maxInt64(int64(e.Max), int64(p.Min))) + return nil +} + +// Payload returns the aggregated value for this event +func (e PrecisionTiming) Payload() interface{} { + return e +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e PrecisionTiming) Stats() []string { + return []string{ + fmt.Sprintf("%s.count:%d|c", e.Name, e.Count), + fmt.Sprintf("%s.avg:%.6f|ms", e.Name, float64(int64(e.Value)/e.Count)/1000000), // make sure e.Count != 0 + fmt.Sprintf("%s.min:%.6f|ms", e.Name, e.durationToMs(e.Min)), + fmt.Sprintf("%s.max:%.6f|ms", e.Name, e.durationToMs(e.Max)), + } +} + +// durationToMs converts time.Duration into the corresponding value in milliseconds +func (e PrecisionTiming) durationToMs(x time.Duration) float64 { + return float64(x) / float64(time.Millisecond) +} + +// Key returns the name of this metric +func (e PrecisionTiming) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *PrecisionTiming) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e PrecisionTiming) Type() int { + return EventPrecisionTiming +} + +// TypeString returns a name for this type of metric +func (e PrecisionTiming) TypeString() string { + return "PrecisionTiming" +} + +// String returns a debug-friendly representation of this metric +func (e PrecisionTiming) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Value: %+v}", e.TypeString(), e.Name, e.Payload()) +} diff --git a/lib/statsd/event/precisiontiming_test.go b/lib/statsd/event/precisiontiming_test.go new file mode 100644 index 000000000..8cfc8cf60 --- /dev/null +++ b/lib/statsd/event/precisiontiming_test.go @@ -0,0 +1,21 @@ +package event + +import ( + "reflect" + "testing" + "time" +) + +func TestPrecisionTimingUpdate(t *testing.T) { + e1 := NewPrecisionTiming("test", 5*time.Microsecond) + e2 := NewPrecisionTiming("test", 3*time.Microsecond) + e3 := NewPrecisionTiming("test", 7*time.Microsecond) + e1.Update(e2) + e1.Update(e3) + + expected := []string{"test.count:3|c", "test.avg:0.005000|ms", "test.min:0.003000|ms", "test.max:0.007000|ms"} + actual := e1.Stats() + if !reflect.DeepEqual(expected, actual) { + t.Errorf("did not receive all metrics: Expected: %T %v, Actual: %T %v ", expected, expected, actual, actual) + } +} diff --git a/lib/statsd/event/timing.go b/lib/statsd/event/timing.go new file mode 100644 index 000000000..6e97e4901 --- /dev/null +++ b/lib/statsd/event/timing.go @@ -0,0 +1,88 @@ +package event + +import "fmt" + +// Timing keeps min/max/avg information about a timer over a certain interval +type Timing struct { + Name string + Min int64 + Max int64 + Value int64 + Count int64 +} + +// NewTiming is a factory for a Timing event, setting the Count to 1 to prevent div_by_0 errors +func NewTiming(k string, delta int64) *Timing { + return &Timing{Name: k, Min: delta, Max: delta, Value: delta, Count: 1} +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *Timing) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + p := e2.Payload().(map[string]int64) + e.Count += p["cnt"] + e.Value += p["val"] + e.Min = minInt64(e.Min, p["min"]) + e.Max = maxInt64(e.Max, p["max"]) + return nil +} + +// Payload returns the aggregated value for this event +func (e Timing) Payload() interface{} { + return map[string]int64{ + "min": e.Min, + "max": e.Max, + "val": e.Value, + "cnt": e.Count, + } +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e Timing) Stats() []string { + return []string{ + fmt.Sprintf("%s.count:%d|c", e.Name, e.Count), + fmt.Sprintf("%s.avg:%d|ms", e.Name, int64(e.Value/e.Count)), // make sure e.Count != 0 + fmt.Sprintf("%s.min:%d|ms", e.Name, e.Min), + fmt.Sprintf("%s.max:%d|ms", e.Name, e.Max), + } +} + +// Key returns the name of this metric +func (e Timing) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *Timing) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e Timing) Type() int { + return EventTiming +} + +// TypeString returns a name for this type of metric +func (e Timing) TypeString() string { + return "Timing" +} + +// String returns a debug-friendly representation of this metric +func (e Timing) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Value: %+v}", e.TypeString(), e.Name, e.Payload()) +} + +func minInt64(v1, v2 int64) int64 { + if v1 <= v2 { + return v1 + } + return v2 +} +func maxInt64(v1, v2 int64) int64 { + if v1 >= v2 { + return v1 + } + return v2 +} diff --git a/lib/statsd/event/timing_test.go b/lib/statsd/event/timing_test.go new file mode 100644 index 000000000..4ea77d74b --- /dev/null +++ b/lib/statsd/event/timing_test.go @@ -0,0 +1,20 @@ +package event + +import ( + "reflect" + "testing" +) + +func TestTimingUpdate(t *testing.T) { + e1 := NewTiming("test", 5) + e2 := NewTiming("test", 3) + e3 := NewTiming("test", 7) + e1.Update(e2) + e1.Update(e3) + + expected := []string{"test.count:3|c", "test.avg:5|ms", "test.min:3|ms", "test.max:7|ms"} + actual := e1.Stats() + if !reflect.DeepEqual(expected, actual) { + t.Errorf("did not receive all metrics: Expected: %T %v, Actual: %T %v ", expected, expected, actual, actual) + } +} diff --git a/lib/statsd/event/total.go b/lib/statsd/event/total.go new file mode 100644 index 000000000..6c3ca3ec7 --- /dev/null +++ b/lib/statsd/event/total.go @@ -0,0 +1,53 @@ +package event + +import "fmt" + +// Total represents a metric that is continously increasing, e.g. read operations since boot +type Total struct { + Name string + Value int64 +} + +// Update the event with metrics coming from a new one of the same type and with the same key +func (e *Total) Update(e2 Event) error { + if e.Type() != e2.Type() { + return fmt.Errorf("statsd event type conflict: %s vs %s ", e.String(), e2.String()) + } + e.Value += e2.Payload().(int64) + return nil +} + +// Payload returns the aggregated value for this event +func (e Total) Payload() interface{} { + return e.Value +} + +// Stats returns an array of StatsD events as they travel over UDP +func (e Total) Stats() []string { + return []string{fmt.Sprintf("%s:%d|t", e.Name, e.Value)} +} + +// Key returns the name of this metric +func (e Total) Key() string { + return e.Name +} + +// SetKey sets the name of this metric +func (e *Total) SetKey(key string) { + e.Name = key +} + +// Type returns an integer identifier for this type of metric +func (e Total) Type() int { + return EventTotal +} + +// TypeString returns a name for this type of metric +func (e Total) TypeString() string { + return "Total" +} + +// String returns a debug-friendly representation of this metric +func (e Total) String() string { + return fmt.Sprintf("{Type: %s, Key: %s, Value: %d}", e.TypeString(), e.Name, e.Value) +} diff --git a/lib/statsd/interface.go b/lib/statsd/interface.go new file mode 100644 index 000000000..83edeb673 --- /dev/null +++ b/lib/statsd/interface.go @@ -0,0 +1,22 @@ +package statsd + +import "time" + +// Statsd is an interface to a StatsD client (buffered/unbuffered) +type Statsd interface { + CreateSocket() error + CreateTCPSocket() error + Close() error + Incr(stat string, count int64) error + Decr(stat string, count int64) error + Timing(stat string, delta int64) error + PrecisionTiming(stat string, delta time.Duration) error + Gauge(stat string, value int64) error + GaugeDelta(stat string, value int64) error + Absolute(stat string, value int64) error + Total(stat string, value int64) error + + FGauge(stat string, value float64) error + FGaugeDelta(stat string, value float64) error + FAbsolute(stat string, value float64) error +} diff --git a/lib/statsd/noopclient.go b/lib/statsd/noopclient.go new file mode 100644 index 000000000..e432a7502 --- /dev/null +++ b/lib/statsd/noopclient.go @@ -0,0 +1,80 @@ +package statsd + +//@author https://github.com/wyndhblb/statsd + +import ( + "time" +) + +// NoopClient implements a "no-op" statsd in case there is no statsd server +type NoopClient struct{} + +// CreateSocket does nothing +func (s NoopClient) CreateSocket() error { + return nil +} + +// CreateTCPSocket does nothing +func (s NoopClient) CreateTCPSocket() error { + return nil +} + +// Close does nothing +func (s NoopClient) Close() error { + return nil +} + +// Incr does nothing +func (s NoopClient) Incr(stat string, count int64) error { + return nil +} + +// Decr does nothing +func (s NoopClient) Decr(stat string, count int64) error { + return nil +} + +// Timing does nothing +func (s NoopClient) Timing(stat string, count int64) error { + return nil +} + +// PrecisionTiming does nothing +func (s NoopClient) PrecisionTiming(stat string, delta time.Duration) error { + return nil +} + +// Gauge does nothing +func (s NoopClient) Gauge(stat string, value int64) error { + return nil +} + +// GaugeDelta does nothing +func (s NoopClient) GaugeDelta(stat string, value int64) error { + return nil +} + +// Absolute does nothing +func (s NoopClient) Absolute(stat string, value int64) error { + return nil +} + +// Total does nothing +func (s NoopClient) Total(stat string, value int64) error { + return nil +} + +// FGauge does nothing +func (s NoopClient) FGauge(stat string, value float64) error { + return nil +} + +// FGaugeDelta does nothing +func (s NoopClient) FGaugeDelta(stat string, value float64) error { + return nil +} + +// FAbsolute does nothing +func (s NoopClient) FAbsolute(stat string, value float64) error { + return nil +} diff --git a/lib/tencentcloud/LICENSE b/lib/tencentcloud/LICENSE new file mode 100644 index 000000000..25cc8dd5b --- /dev/null +++ b/lib/tencentcloud/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2020 Levi.Lu +Copyright (c) 2024 nezhahq + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/lib/tencentcloud/README.md b/lib/tencentcloud/README.md new file mode 100644 index 000000000..bbfb817b0 --- /dev/null +++ b/lib/tencentcloud/README.md @@ -0,0 +1,23 @@ +# TencentCloud DNSPod for `libdns` + +This package implements the [libdns](https://github.com/libdns/libdns) interfaces for the [TencentCloud DNSPod API](https://www.tencentcloud.com/zh/document/api/1157/49025) + +## Code example + +```go +import "github.com/libdns/tencentcloud" +provider := &tencentcloud.Provider{ + SecretId: "YOUR_Secret_ID", + SecretKey: "YOUR_Secret_Key", +} +``` + +## Security Credentials + +To authenticate you need to supply a [TencentCloud API Key](https://console.tencentcloud.com/cam/capi). + +## Other instructions + +`libdns/tencentcloud` is based on the new version of Tencentcloud api, uses secret Id and key as authentication methods, supports permission settings, and supports DNSPod international version. + +`libdns/dnspod` is based on the old version of dnspod.cn api, uses token as the authentication method, does not support permission settings, and does not support DNSPod international version. diff --git a/lib/tencentcloud/client.go b/lib/tencentcloud/client.go new file mode 100644 index 000000000..30a3aa5df --- /dev/null +++ b/lib/tencentcloud/client.go @@ -0,0 +1,201 @@ +package tencentcloud + +import ( + "context" + "encoding/json" + "io" + "net/http" + "strings" + "time" + + "github.com/libdns/libdns" +) + +const ( + endpoint = "https://dnspod.tencentcloudapi.com" + + DescribeRecordList = "DescribeRecordList" + CreateRecord = "CreateRecord" + ModifyRecord = "ModifyRecord" + DeleteRecord = "DeleteRecord" +) + +func (p *Provider) listRecords(ctx context.Context, zone string) ([]libdns.Record, error) { + domain := strings.TrimSuffix(zone, ".") + + requestData := FindRecordRequest{ + Domain: domain, + RecordLine: "默认", + } + + payload, err := json.Marshal(requestData) + if err != nil { + return nil, err + } + + resp, err := p.sendRequest(ctx, DescribeRecordList, string(payload)) + if err != nil { + return nil, err + } + + var response Response + if err = json.Unmarshal(resp, &response); err != nil { + return nil, err + } + + list := make([]libdns.Record, 0, len(response.Response.RecordList)) + for _, txRecord := range response.Response.RecordList { + rr := record{ + Type: txRecord.Type, + Name: txRecord.Name, + Value: txRecord.Value, + TTL: time.Duration(txRecord.TTL) * time.Second, + } + libdnsRecord, err := rr.libdnsRecord() + if err != nil { + return nil, err + } + list = append(list, libdnsRecord) + } + + return list, nil +} + +func (p *Provider) createRecord(ctx context.Context, zone string, record libdns.Record) error { + domain := strings.TrimSuffix(zone, ".") + r := fromLibdnsRecord(record) + requestData := CreateModifyRecordRequest{ + Domain: domain, + SubDomain: r.Name, + RecordType: r.Type, + RecordLine: "默认", + Value: r.Value, + TTL: int64(r.TTL.Seconds()), + } + + payload, err := json.Marshal(requestData) + if err != nil { + return err + } + + resp, err := p.sendRequest(ctx, CreateRecord, string(payload)) + if err != nil { + return err + } + + var response Response + if err := json.Unmarshal(resp, &response); err != nil { + return err + } + + if response.Response.RecordId == 0 { + return ErrNotValid + } + + return nil +} + +func (p *Provider) modifyRecord(ctx context.Context, id uint64, zone string, record libdns.Record) error { + domain := strings.TrimSuffix(zone, ".") + r := fromLibdnsRecord(record) + requestData := CreateModifyRecordRequest{ + Domain: domain, + SubDomain: r.Name, + RecordType: r.Type, + RecordLine: "默认", + Value: r.Value, + TTL: int64(r.TTL.Seconds()), + RecordId: id, + } + + payload, err := json.Marshal(requestData) + if err != nil { + return err + } + + _, err = p.sendRequest(ctx, ModifyRecord, string(payload)) + return err +} + +func (p *Provider) deleteRecord(ctx context.Context, id uint64, zone string) error { + domain := strings.TrimSuffix(zone, ".") + + requestData := DeleteRecordRequest{ + Domain: domain, + RecordId: id, + } + + payload, err := json.Marshal(requestData) + if err != nil { + return err + } + + _, err = p.sendRequest(ctx, DeleteRecord, string(payload)) + return err +} + +func (p *Provider) findRecord(ctx context.Context, zone string, record libdns.Record) (uint64, error) { + domain := strings.TrimSuffix(zone, ".") + r := fromLibdnsRecord(record) + requestData := FindRecordRequest{ + Domain: domain, + RecordType: r.Type, + RecordLine: "默认", + Subdomain: r.Name, + Limit: 3000, + } + payload, err := json.Marshal(requestData) + if err != nil { + return 0, err + } + + resp, err := p.sendRequest(ctx, DescribeRecordList, string(payload)) + if err != nil { + return 0, err + } + + var response Response + if err = json.Unmarshal(resp, &response); err != nil { + return 0, err + } + var recordId uint64 + for _, item := range response.Response.RecordList { + if item.Name == r.Name && item.Type == r.Type { + if r.Value != "" && item.Value != r.Value { + continue + } + recordId = uint64(item.RecordId) + break + } + } + + if recordId == 0 { + return 0, ErrRecordNotFound + } + + return recordId, nil +} + +func (p *Provider) sendRequest(ctx context.Context, action string, data string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, "POST", endpoint, strings.NewReader(data)) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-TC-Version", "2021-03-23") + + SignRequest(p.SecretId, p.SecretKey, req, action, data) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return body, nil +} diff --git a/lib/tencentcloud/provider.go b/lib/tencentcloud/provider.go new file mode 100644 index 000000000..c4f02c0d3 --- /dev/null +++ b/lib/tencentcloud/provider.go @@ -0,0 +1,63 @@ +package tencentcloud + +import ( + "context" + "errors" + + "github.com/libdns/libdns" +) + +func (p *Provider) GetRecords(ctx context.Context, zone string) ([]libdns.Record, error) { + return p.listRecords(ctx, zone) +} + +func (p *Provider) AppendRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { + for _, record := range records { + if err := p.createRecord(ctx, zone, record); err != nil { + return nil, err + } + } + + return records, nil +} + +func (p *Provider) SetRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { + for _, record := range records { + id, err := p.findRecord(ctx, zone, record) + if err != nil { + if errors.Is(err, ErrRecordNotFound) { + if err = p.createRecord(ctx, zone, record); err != nil { + return nil, err + } + continue + } + } + if err = p.modifyRecord(ctx, id, zone, record); err != nil { + return nil, err + } + } + + return records, nil +} + +func (p *Provider) DeleteRecords(ctx context.Context, zone string, records []libdns.Record) ([]libdns.Record, error) { + for _, record := range records { + id, err := p.findRecord(ctx, zone, record) + if err != nil { + return nil, err + } + if err := p.deleteRecord(ctx, id, zone); err != nil { + return nil, err + } + } + + return records, nil +} + +// Interface guards +var ( + _ libdns.RecordGetter = (*Provider)(nil) + _ libdns.RecordAppender = (*Provider)(nil) + _ libdns.RecordSetter = (*Provider)(nil) + _ libdns.RecordDeleter = (*Provider)(nil) +) diff --git a/lib/tencentcloud/provider_test.go b/lib/tencentcloud/provider_test.go new file mode 100644 index 000000000..dbb2f6339 --- /dev/null +++ b/lib/tencentcloud/provider_test.go @@ -0,0 +1,61 @@ +package tencentcloud + +import ( + "context" + "net/netip" + "os" + "testing" + + "github.com/libdns/libdns" +) + +var provider = &Provider{ + SecretId: os.Getenv("TC_SECRET_ID"), + SecretKey: os.Getenv("TC_SECRET_KEY"), +} + +var ( + zone = os.Getenv("TC_ZONE") + name = os.Getenv("TC_NAME") + value = os.Getenv("TC_VALUE") +) + +func requireTencentCloudEnv(t *testing.T) { + t.Helper() + + if provider.SecretId == "" || provider.SecretKey == "" || zone == "" || name == "" || value == "" { + t.Skip("skipping Tencent Cloud integration test: required TC_* env vars are not set") + } +} + +func TestSetRecords(t *testing.T) { + requireTencentCloudEnv(t) + + netip, err := netip.ParseAddr(value) + if err != nil { + t.Fatalf("parse error: %v", err) + } + _, err = provider.SetRecords(context.Background(), zone, []libdns.Record{ + libdns.Address{ + Name: name, + IP: netip, + }, + }) + if err != nil { + t.Fatalf("SetRecords: %v", err) + } +} + +func TestGetRecords(t *testing.T) { + requireTencentCloudEnv(t) + + records, err := provider.GetRecords(context.Background(), zone) + if err != nil { + t.Fatalf("GetRecords: %v", err) + } + for _, record := range records { + rr := record.RR() + t.Logf("RecordType: %s, Name: %s, Data: %s", + rr.Type, rr.Name, rr.Data) + } +} diff --git a/lib/tencentcloud/signer.go b/lib/tencentcloud/signer.go new file mode 100644 index 000000000..e4fd6f278 --- /dev/null +++ b/lib/tencentcloud/signer.go @@ -0,0 +1,66 @@ +package tencentcloud + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http" + "strconv" + "strings" + "time" +) + +// SignRequest https://github.com/jeessy2/ddns-go/blob/master/util/tencent_cloud_signer.go +func SignRequest(secretId string, secretKey string, r *http.Request, action string, payload string) { + algorithm := "TC3-HMAC-SHA256" + service := "dnspod" + host := writeString(service, ".tencentcloudapi.com") + timestamp := time.Now().Unix() + timestampStr := strconv.FormatInt(timestamp, 10) + + // 步骤 1：拼接规范请求串 + canonicalHeaders := writeString("content-type:application/json\nhost:", host, "\nx-tc-action:", strings.ToLower(action), "\n") + signedHeaders := "content-type;host;x-tc-action" + hashedRequestPayload := sha256hex(payload) + canonicalRequest := writeString("POST\n/\n\n", canonicalHeaders, "\n", signedHeaders, "\n", hashedRequestPayload) + + // 步骤 2：拼接待签名字符串 + date := time.Unix(timestamp, 0).UTC().Format("2006-01-02") + credentialScope := writeString(date, "/", service, "/tc3_request") + hashedCanonicalRequest := sha256hex(canonicalRequest) + string2sign := writeString(algorithm, "\n", timestampStr, "\n", credentialScope, "\n", hashedCanonicalRequest) + + // 步骤 3：计算签名 + secretDate := hmacsha256(date, writeString("TC3", secretKey)) + secretService := hmacsha256(service, secretDate) + secretSigning := hmacsha256("tc3_request", secretService) + signature := hex.EncodeToString([]byte(hmacsha256(string2sign, secretSigning))) + + // 步骤 4：拼接 Authorization + authorization := writeString(algorithm, " Credential=", secretId, "/", credentialScope, ", SignedHeaders=", signedHeaders, ", Signature=", signature) + + r.Header.Set("Authorization", authorization) + r.Header.Set("Host", host) + r.Header.Set("X-TC-Action", action) + r.Header.Set("X-TC-Timestamp", timestampStr) +} + +func sha256hex(s string) string { + b := sha256.Sum256([]byte(s)) + return hex.EncodeToString(b[:]) +} + +func hmacsha256(s, key string) string { + hashed := hmac.New(sha256.New, []byte(key)) + hashed.Write([]byte(s)) + return string(hashed.Sum(nil)) +} + +func writeString(strs ...string) string { + var b strings.Builder + for _, str := range strs { + b.WriteString(str) + } + + return b.String() +} diff --git a/lib/tencentcloud/types.go b/lib/tencentcloud/types.go new file mode 100644 index 000000000..8f0d6cc9e --- /dev/null +++ b/lib/tencentcloud/types.go @@ -0,0 +1,98 @@ +package tencentcloud + +import ( + "errors" + "time" + + "github.com/libdns/libdns" +) + +var ErrRecordNotFound = errors.New("record not found") +var ErrNotValid = errors.New("returned value is not valid") + +type Provider struct { + SecretId string + SecretKey string +} + +type CreateModifyRecordRequest struct { + Domain string `json:"Domain"` + SubDomain string `json:"SubDomain,omitempty"` + RecordType string `json:"RecordType,omitempty"` + RecordLine string `json:"RecordLine,omitempty"` + Value string `json:"Value,omitempty"` + TTL int64 `json:"TTL,omitempty"` + RecordId uint64 `json:"RecordId,omitempty"` +} + +type FindRecordRequest struct { + Domain string `json:"Domain"` + RecordType string `json:"RecordType,omitempty"` + RecordLine string `json:"RecordLine,omitempty"` + Subdomain string `json:"Subdomain,omitempty"` + Limit int64 `json:"Limit,omitempty"` +} + +type DeleteRecordRequest struct { + Domain string `json:"Domain"` + RecordId uint64 `json:"RecordId"` +} + +type Response struct { + Response ResponseData `json:"Response"` +} + +type ResponseData struct { + RecordList []RecordInfo `json:"RecordList,omitempty"` + RecordId uint64 `json:"RecordId,omitempty"` + Error *ErrorInfo `json:"Error,omitempty"` +} + +type RecordInfo struct { + RecordId int64 `json:"RecordId"` + Type string `json:"Type"` + Name string `json:"Name"` + Value string `json:"Value"` + TTL int64 `json:"TTL"` +} + +type ErrorInfo struct { + Code string `json:"Code"` + Message string `json:"Message"` +} + +type record struct { + Type string + Name string + Value string + TTL time.Duration +} + +func (r record) libdnsRecord() (libdns.Record, error) { + return libdns.RR{ + Type: r.Type, + Name: r.Name, + Data: r.Value, + TTL: r.TTL, + }.Parse() +} + +func fromLibdnsRecord(r libdns.Record) record { + rr := r.RR() + + host := rr.Name + if host == "@" { + host = "" + } + + if rr.TTL == 0 { + rr.TTL = 600 + } + + return record{ + Type: rr.Type, + Name: host, + Value: rr.Data, + TTL: rr.TTL, + } +} diff --git a/modules/elastic/adapter/easysearch/v1.go b/modules/elastic/adapter/easysearch/v1.go index 36b89225e..e8aa4a5a4 100644 --- a/modules/elastic/adapter/easysearch/v1.go +++ b/modules/elastic/adapter/easysearch/v1.go @@ -46,7 +46,7 @@ func (c *APIV1) StartReplication(followIndex string, body []byte) error { } if resp.StatusCode != http.StatusOK { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -58,7 +58,7 @@ func (c *APIV1) StopReplication(indexName string, body []byte) error { } if resp.StatusCode != http.StatusOK { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -74,7 +74,7 @@ func (c *APIV1) PauseReplication(followIndex string, body []byte) error { } if resp.StatusCode != http.StatusOK { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -90,7 +90,7 @@ func (c *APIV1) ResumeReplication(followIndex string, body []byte) error { } if resp.StatusCode != http.StatusOK { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -102,7 +102,7 @@ func (c *APIV1) GetReplicationStatus(followIndex string) ([]byte, error) { } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } @@ -114,7 +114,7 @@ func (c *APIV1) GetReplicationFollowerStats(followIndex string) ([]byte, error) } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } @@ -126,7 +126,7 @@ func (c *APIV1) CreateAutoFollowReplication(autoFollowPatternName string, body [ } if resp.StatusCode != http.StatusOK { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -138,7 +138,7 @@ func (c *APIV1) DeleteAutoFollowReplication(autoFollowPatternName string, body [ return err } if resp.StatusCode != http.StatusOK { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -150,7 +150,7 @@ func (c *APIV1) GetAutoFollowStats(autoFollowPatternName string) ([]byte, error) } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } @@ -163,7 +163,7 @@ func (c *APIV1) GetUser(username string) ([]byte, error) { } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } @@ -174,7 +174,7 @@ func (c *APIV1) GetUsers() ([]byte, error) { return nil, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } @@ -185,7 +185,7 @@ func (c *APIV1) DeleteUser(username string) error { return err } if resp.StatusCode != http.StatusOK { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -196,7 +196,7 @@ func (c *APIV1) PutUser(username string, body []byte) error { return err } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -207,7 +207,7 @@ func (c *APIV1) GetRole(roleName string) ([]byte, error) { return nil, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } @@ -218,7 +218,7 @@ func (c *APIV1) GetRoles() ([]byte, error) { return nil, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } @@ -229,7 +229,7 @@ func (c *APIV1) DeleteRole(roleName string) error { return err } if resp.StatusCode != http.StatusOK { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -241,7 +241,7 @@ func (c *APIV1) PutRole(roleName string, body []byte) error { } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil } @@ -252,7 +252,7 @@ func (c *APIV1) GetPrivileges() ([]byte, error) { return nil, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } diff --git a/modules/elastic/adapter/elasticsearch/V6.6.go b/modules/elastic/adapter/elasticsearch/V6.6.go index f228bba48..6c17bbddb 100644 --- a/modules/elastic/adapter/elasticsearch/V6.6.go +++ b/modules/elastic/adapter/elasticsearch/V6.6.go @@ -66,7 +66,7 @@ func (s *ESAPIV6_6) GetILMPolicy(target string) (map[string]interface{}, error) } if resp.StatusCode != 200 { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } data := map[string]interface{}{} @@ -87,7 +87,7 @@ func (s *ESAPIV6_6) PutILMPolicy(target string, policyConfig []byte) error { } if resp.StatusCode != 200 { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil @@ -105,7 +105,7 @@ func (s *ESAPIV6_6) DeleteILMPolicy(target string) error { } if resp.StatusCode != 200 { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil diff --git a/modules/elastic/adapter/elasticsearch/v0.go b/modules/elastic/adapter/elasticsearch/v0.go index f75251e9c..335d8215a 100755 --- a/modules/elastic/adapter/elasticsearch/v0.go +++ b/modules/elastic/adapter/elasticsearch/v0.go @@ -1254,7 +1254,7 @@ func (s *ESAPIV0) UpdateIndexSettings(name string, settings map[string]interface result, err := s.Request(nil, util.Verb_PUT, url, body.Bytes()) errReason, _ := jsonparser.GetString(result.Body, "error", "reason") if errReason != "" { - return fmt.Errorf(errReason) + return fmt.Errorf("%s", errReason) } return err @@ -1277,7 +1277,7 @@ func (s *ESAPIV0) UpdateMapping(indexName string, docType string, mappings []byt panic(err) } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil @@ -1430,7 +1430,7 @@ func (c *ESAPIV0) GetTemplate(templateName string) (map[string]interface{}, erro } if resp.StatusCode != 200 { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } data := map[string]interface{}{} @@ -1739,7 +1739,7 @@ func (c *ESAPIV0) Alias(body []byte) error { return err } if res.StatusCode != http.StatusOK { - return fmt.Errorf(string(res.Body)) + return fmt.Errorf("%s", res.Body) } return nil } @@ -1894,7 +1894,7 @@ func (c *ESAPIV0) UpdateClusterSettings(body []byte) error { } if resp.StatusCode != http.StatusOK { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil @@ -1907,7 +1907,7 @@ func (c *ESAPIV0) GetRemoteInfo() ([]byte, error) { return nil, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil @@ -2017,7 +2017,7 @@ func (c *ESAPIV0) Flush(indexName string) ([]byte, error) { return nil, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } @@ -2048,7 +2048,7 @@ func (c *ESAPIV0) ClusterAllocationExplain(ctx context.Context, body []byte, par return nil, err } if resp.StatusCode != 200 { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } @@ -2060,7 +2060,7 @@ func (c *ESAPIV0) CatAllocation(ctx context.Context) ([]elastic.CatAllocationRes return nil, err } if resp.StatusCode != 200 { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } data := []elastic.CatAllocationResponse{} err = json.Unmarshal(resp.Body, &data) diff --git a/modules/elastic/adapter/elasticsearch/v7.go b/modules/elastic/adapter/elasticsearch/v7.go index 54a38faae..9b0c51c32 100755 --- a/modules/elastic/adapter/elasticsearch/v7.go +++ b/modules/elastic/adapter/elasticsearch/v7.go @@ -412,7 +412,7 @@ func (c *ESAPIV7) UpdateMapping(indexName string, docType string, mappings []byt panic(err) } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, err @@ -430,7 +430,7 @@ func (c *ESAPIV7) ScriptExists(scriptName string) (bool, error) { return false, err } if resp.StatusCode != http.StatusOK { - return false, fmt.Errorf(string(resp.Body)) + return false, fmt.Errorf("%s", resp.Body) } return true, nil } @@ -447,7 +447,7 @@ func (c *ESAPIV7) PutScript(scriptName string, script []byte) ([]byte, error) { return nil, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } diff --git a/modules/elastic/adapter/elasticsearch/v8.go b/modules/elastic/adapter/elasticsearch/v8.go index b82560b8a..c6f0dc0a4 100644 --- a/modules/elastic/adapter/elasticsearch/v8.go +++ b/modules/elastic/adapter/elasticsearch/v8.go @@ -319,7 +319,7 @@ func (c *ESAPIV8) Flush(indexName string) ([]byte, error) { return nil, err } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } return resp.Body, nil } diff --git a/modules/elastic/adapter/opensearch/v1.go b/modules/elastic/adapter/opensearch/v1.go index 7669d26bc..127b61166 100644 --- a/modules/elastic/adapter/opensearch/v1.go +++ b/modules/elastic/adapter/opensearch/v1.go @@ -50,7 +50,7 @@ func (s *APIV1) GetILMPolicy(target string) (map[string]interface{}, error) { } if resp.StatusCode != 200 { - return nil, fmt.Errorf(string(resp.Body)) + return nil, fmt.Errorf("%s", resp.Body) } data := map[string]interface{}{} @@ -71,7 +71,7 @@ func (s *APIV1) PutILMPolicy(target string, policyConfig []byte) error { } if resp.StatusCode != 200 && resp.StatusCode != 201 { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil @@ -89,7 +89,7 @@ func (s *APIV1) DeleteILMPolicy(target string) error { } if resp.StatusCode != 200 { - return fmt.Errorf(string(resp.Body)) + return fmt.Errorf("%s", resp.Body) } return nil diff --git a/modules/stats/simple_test.go b/modules/stats/simple_test.go deleted file mode 100644 index 60487b38f..000000000 --- a/modules/stats/simple_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (C) INFINI Labs & INFINI LIMITED. -// -// The INFINI Framework is offered under the GNU Affero General Public License v3.0 -// and as commercial software. -// -// For commercial licensing, contact us at: -// - Website: infinilabs.com -// - Email: hello@infini.ltd -// -// Open Source licensed under AGPL V3: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -/* Copyright © INFINI LTD. All rights reserved. - * Web: https://infinilabs.com - * Email: hello#infini.ltd */ - -package stats - -import ( - "fmt" - "os" - "regexp" - "runtime" - "strings" - "testing" -) - -func TestGoroutinesInfo(t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - buf := make([]byte, 2<<20) - n := runtime.Stack(buf, true) - - stacks := strings.Split(string(buf[:n]), "\n\n") - grouped := make(map[string]int) - patternMem, err := regexp.Compile("\\+?0x[\\d\\w]+") - if err != nil { - panic(err) - } - patternID, err := regexp.Compile("^goroutine \\d+") - if err != nil { - panic(err) - } - - patternNewID, err := regexp.Compile("^goroutine ID") - if err != nil { - panic(err) - } - for _, stack := range stacks { - newStack := patternMem.ReplaceAll([]byte(stack), []byte("_address_")) - newStack = patternID.ReplaceAll([]byte(newStack), []byte("goroutine ID")) - grouped[string(newStack)]++ - } - - for funcPath, count := range grouped { - str := patternNewID.ReplaceAllString(funcPath, fmt.Sprintf("%v same instance of goroutines", count)) - fmt.Printf("%v\n", str) - } -} diff --git a/plugins/badger/module_test.go b/plugins/badger/module_test.go index 0e7fbc6b0..83a162895 100755 --- a/plugins/badger/module_test.go +++ b/plugins/badger/module_test.go @@ -28,10 +28,8 @@ package badger import ( - "fmt" "os" "testing" - "time" "github.com/stretchr/testify/assert" . "infini.sh/framework/core/env" @@ -55,37 +53,6 @@ func Test(t *testing.T) { b, _ := filter.CheckThenAdd(filterKey, []byte("key")) assert.Equal(t, false, b) - //err=filter.Add(filterKey,[]byte("key")) - //fmt.Println(err) - //ok:=filter.Exists(filterKey,[]byte("key")) - //fmt.Println(ok) - - //Memory pressure test - for i := 0; i < 1; i++ { - go run(i, t) - } - - time.Sleep(10 * time.Second) - - //For BoltDB KV filter, 19k unique will consume 100MB memory, 40K:230MB -} - -func run(seed int, t *testing.T) { - if os.Getenv("CI") == "true" { - t.Skip("Skipping in CI environment") - } - for i := 0; i < 100000000; i++ { - fmt.Println(i) - k := fmt.Sprintf("key-%v-%v", seed, i) - b := filter.Exists(filterKey, []byte(k)) - assert.Equal(t, false, b) - b, _ = filter.CheckThenAdd(filterKey, []byte(k)) - assert.Equal(t, false, b) - b = filter.Exists(filterKey, []byte(k)) - assert.Equal(t, true, b) - if !b { - fmt.Print("not exists") - } - } - fmt.Println("done", seed) + b = filter.Exists(filterKey, []byte("key")) + assert.Equal(t, true, b) } diff --git a/plugins/smtp/smtp.go b/plugins/smtp/smtp.go index f02f3fc49..8c4e0119e 100644 --- a/plugins/smtp/smtp.go +++ b/plugins/smtp/smtp.go @@ -34,7 +34,6 @@ import ( "time" log "github.com/cihub/seelog" - "github.com/gopkg.in/gomail.v2" "infini.sh/framework/core/config" "infini.sh/framework/core/errors" "infini.sh/framework/core/global" @@ -43,6 +42,7 @@ import ( "infini.sh/framework/core/queue" "infini.sh/framework/core/util" "infini.sh/framework/lib/fasttemplate" + "infini.sh/framework/lib/gomail" ) type SMTPProcessor struct { diff --git a/plugins/stats_statsd/statsd.go b/plugins/stats_statsd/statsd.go index e7df21664..bd8a35c18 100755 --- a/plugins/stats_statsd/statsd.go +++ b/plugins/stats_statsd/statsd.go @@ -25,12 +25,13 @@ package statsd import ( "fmt" + "time" + log "github.com/cihub/seelog" - "github.com/quipo/statsd" "infini.sh/framework/core/env" "infini.sh/framework/core/errors" "infini.sh/framework/core/stats" - "time" + "infini.sh/framework/lib/statsd" ) type StatsDConfig struct { From e9a7a2d56ed7ca44b1004abf2faf7d7e2ccd267e Mon Sep 17 00:00:00 2001 From: Hardy Date: Mon, 25 May 2026 10:32:34 +0800 Subject: [PATCH 062/137] fix: shared format target for non-module repositories (#361) * fix: make format work without go modules Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * ci: rerun title check on pull request edits Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/commit-message-check.yml | 1 + Makefile | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/commit-message-check.yml b/.github/workflows/commit-message-check.yml index 45af0b0f2..331ae5ce7 100644 --- a/.github/workflows/commit-message-check.yml +++ b/.github/workflows/commit-message-check.yml @@ -1,6 +1,7 @@ name: 'commit-message-check' on: pull_request: + types: [opened, synchronize, reopened, edited] jobs: check-commit-message: diff --git a/Makefile b/Makefile index daeada38f..ac10d1d9a 100755 --- a/Makefile +++ b/Makefile @@ -242,7 +242,10 @@ cross-build-all-platform: clean config build-bsd build-linux build-darwin build- format: @echo "formatting code" - $(GO) fmt $$($(GO) list ./...) + find . -type f -name '*.go' \ + -not -path './vendor/*' \ + -not -path './.git/*' \ + -exec gofmt -w {} + test: config $(GOTEST) -v $(GOFLAGS) -timeout 30m ./... From 5f3bb5d7f5d3b66166e536900cfe0e789c158254 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Mon, 25 May 2026 14:55:47 +0800 Subject: [PATCH 063/137] refactor: update gopsutil to v4, add overall host metrics (#281) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: upgrade gopsutil from v3.24.5 to v4.26.3 - Update all import paths from github.com/shirou/gopsutil/v3 to v4 across 10 source files (disk, cpu, memory, network, stats, app, status, and host/sys_info) - Update go.mod dependency to gopsutil/v4 v4.26.3 - Bump minimum Go version to 1.24.0 (required by gopsutil v4) - Update transitive dependencies (go-sysconf, numcpus, perfstat, sys) Key improvement: disk.IOCounters() on darwin/macOS is now implemented via IOKit in gopsutil v4, resolving the 'not implemented yet' error that previously occurred on macOS. The graceful degradation code in disk.go is preserved as a safety net for unsupported platforms. Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/35abcace-ca46-4a40-8c8e-85acd8df15e2 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * Upgrade gopsutil v3 → v4: resolve darwin disk.IOCounters "not implemented yet" Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/35abcace-ca46-4a40-8c8e-85acd8df15e2 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * feat: add overall system utilization metric with health status Add a new 'host/overall' metric collector that computes composite system health by evaluating CPU, memory, disk capacity, and disk I/O utilization. Each subsystem is classified as green/yellow/red based on configurable thresholds (default: 70% yellow, 90% red). The overall status reflects the worst-performing subsystem, identifying the bottleneck. Event payload includes per-subsystem breakdown with status and percentage, plus the top-level status and bottleneck indicator for dashboard display. New types in core/host: - SubsystemHealth: per-subsystem status - OverallStatus: composite health with bottleneck - ClassifyHealth(): threshold-based health classification - HealthPriority(): health comparison helper Configuration (under metrics.overall): - enabled: true/false - yellow_threshold: percentage (default 70) - red_threshold: percentage (default 90) Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/4eaec09e-6e0d-4805-bcfa-cc88c0a3a028 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * refactor: make disk IO interval configurable instead of hardcoded Address code review feedback: extract the 10s interval constant into a configurable IntervalSeconds field (default: 10) on the Metric struct. Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/4eaec09e-6e0d-4805-bcfa-cc88c0a3a028 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * chore: restore config/generated.go to original state Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/4eaec09e-6e0d-4805-bcfa-cc88c0a3a028 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * plan: add network throughput to overall utilization metric Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/35d0ec62-94eb-4e02-a560-9dc9d927e75b Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * feat: add network throughput to overall metric, remove status/bottleneck fields Remove status/bottleneck/threshold classification from the overall utilization metric - the front layer handles green/yellow/red display logic. Add network throughput (bytes/sec in+out) to complete the subsystem coverage: CPU, memory, disk capacity, disk I/O, and network. Remove now-unused types from core/host/host.go: SubsystemHealth, OverallStatus, ClassifyHealth, HealthPriority, DefaultYellowThreshold, DefaultRedThreshold. Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/35d0ec62-94eb-4e02-a560-9dc9d927e75b Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * feat: add network to overall metric, simplify to raw values only Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/35d0ec62-94eb-4e02-a560-9dc9d927e75b Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * refactor: change network metric from bytes/sec to used_percent Network utilization is now reported as a percentage of configured bandwidth (network_bandwidth_mbps, default 1000 Mbps). Uses max(in, out) throughput relative to bandwidth, consistent with all other subsystems reporting used_percent in the overall metric. Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/bb90d5d0-209f-47c0-b000-9550e3005071 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * Remove status/bottleneck from host/overall metric, emit only utilization percentages Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/bb90d5d0-209f-47c0-b000-9550e3005071 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * chore: remove config/generated.go from git tracking The file is already in .gitignore but was still being tracked. This removes it from the index so future changes are properly ignored. Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/314530fc-4e7a-4937-be8f-3aa2679ba12a Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * revert: restore config/generated.go with placeholder values Restore the file with N/A placeholder values instead of removing it from tracking entirely. The file remains in .gitignore so local build-regenerated values won't be accidentally committed. Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/5c37d476-1ae1-4466-963d-56cec89b93a3 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * Restore config/generated.go with placeholder values and remove from git tracking Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/5c37d476-1ae1-4466-963d-56cec89b93a3 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * chore: remove config/generated.go from git tracking The file should remain gitignored as it's regenerated during builds with build-specific values (commit hashes, timestamps). Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/4f289f27-76e1-42f7-a474-d0f40cf1c518 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * revert: restore config/generated.go to match main branch Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/2358e99f-f923-489e-9eca-d8a129cc20a4 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * fix: revert gopsutil from v4 to v3 to fix CI build failure Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/ffb7d4c4-0bd7-4667-a1d8-6272e617083a Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * fix: restore gopsutil v4 and update CI to use module mode + Go 1.24 Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/510abe92-44c8-49b4-81fa-cb87020192a8 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * feat: add status and bottleneck fields to overall metrics with configurable thresholds Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/98216f64-3a01-498c-b96c-6e38b2165078 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * feat: auto-detect network interface bandwidth using OS-specific methods Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/f607a8bf-6f7a-4789-bb63-70e325d721e2 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * feat: monitor each disk and network interface independently for bottleneck detection Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/afb21bb5-dc0e-4ba7-ad9d-7cd2a46301a5 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * docs: update release notes for per-device monitoring feature Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/e4b60d91-9418-4dd1-b1c6-48a1d2af1d24 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * refactor: extract default bandwidth as named constant Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/e4b60d91-9418-4dd1-b1c6-48a1d2af1d24 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * chore: remove config/generated.go from tracking (gitignored file) Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/e4b60d91-9418-4dd1-b1c6-48a1d2af1d24 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * docs: remove breaking change note about network_bandwidth_mbps (not merged yet) Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/a9bec310-e8f6-4450-a151-a50df33c6d0e Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * revert: restore config/generated.go Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/824024d6-07bb-4d43-8769-c17fbf573deb Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * ci: restore unit-test workflow toolchain and module mode Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/927e849c-7860-44e7-9ba8-eafc1a665c16 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * chore: revert unintended generated file edits Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/927e849c-7860-44e7-9ba8-eafc1a665c16 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * chore: finalize ci build failure fix validation Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/927e849c-7860-44e7-9ba8-eafc1a665c16 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * chore: drop accidental generated file diffs Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/927e849c-7860-44e7-9ba8-eafc1a665c16 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * ci: switch unit_test workflow to Go modules Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/34eb59e2-0aad-459a-b80b-e72fb45159cd Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * fix: resolve module-mode build failures in CI - plugins/filter_cuckoo: update cuckoofilter API (CuckooFilter→Filter, NewCuckooFilter→NewFilter) to match v0.0.0-20240715131351 - core/security: upgrade jwt import from v3 to v4 (RegisteredClaims, NewNumericDate, VerifyExpiresAt(time.Time) only exist in v4 API); add github.com/golang-jwt/jwt/v4 v4.5.2 to go.mod - lib/guardian/auth/strategies/ldap: update conn interface Close() to Close() error to match go-ldap/v3 v3.4.11 API - modules/elastic/adapter/{elasticsearch,easysearch,opensearch}, cmd/plugin-discovery: fix non-constant format string vet errors by replacing fmt.Errorf(string(x)) with errors.New(string(x)) or fmt.Errorf("%s", ...) and fmt.Fprintf(w, "%s", x) respectively Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/7b7f1cab-bdb5-4278-bf17-7d724989eeb7 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * chore: revert accidental changes to generated files Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/7b7f1cab-bdb5-4278-bf17-7d724989eeb7 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * fix(deps): bump github.com/buger/jsonparser from v1.1.1 to v1.1.2 Addresses GitHub Advisory DoS vulnerability affecting versions <= 1.1.1. v1.1.2 is API-compatible; no source changes required. Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/9803096d-ea8d-4983-beb3-2d26c4f81a5b Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * Merge branch 'main' into copilot/update-gopsutil-to-v4 Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/c26fec12-e5d6-4882-b8d2-c5f8af70919d Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * feat(metrics): add steal/inode/queue/retrans signals to host/overall Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/617e29ca-fc1b-4eea-8061-74a80abae0f7 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * refactor(metrics): clean up cpu collect control flow per review Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/617e29ca-fc1b-4eea-8061-74a80abae0f7 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> * feat(metrics): fold tcp retrans ratio into network status Agent-Logs-Url: https://github.com/infinilabs/framework/sessions/f59fe48f-4192-4258-aca2-3693e6be9d71 Co-authored-by: medcl <64487+medcl@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: medcl <64487+medcl@users.noreply.github.com> Co-authored-by: Medcl Co-authored-by: Copilot --- app.go | 2 +- core/host/sys_info.go | 10 +- go.mod | 23 +- go.sum | 86 +-- lib/status/fs.go | 2 +- modules/metrics/host/cpu/cpu.go | 4 +- modules/metrics/host/disk/disk.go | 2 +- modules/metrics/host/memory/memory.go | 2 +- modules/metrics/host/network/network.go | 2 +- .../metrics/host/network/sockstat_linux.go | 2 +- .../metrics/host/network/sockstat_other.go | 2 +- .../metrics/host/overall/netspeed_darwin.go | 89 +++ .../metrics/host/overall/netspeed_linux.go | 75 +++ .../metrics/host/overall/netspeed_windows.go | 142 ++++ modules/metrics/host/overall/overall.go | 636 ++++++++++++++++++ modules/metrics/metrics.go | 24 + modules/stats/simple.go | 2 +- 17 files changed, 1013 insertions(+), 92 deletions(-) create mode 100644 modules/metrics/host/overall/netspeed_darwin.go create mode 100644 modules/metrics/host/overall/netspeed_linux.go create mode 100644 modules/metrics/host/overall/netspeed_windows.go create mode 100644 modules/metrics/host/overall/overall.go diff --git a/app.go b/app.go index 7b3f74e8b..67a18536a 100755 --- a/app.go +++ b/app.go @@ -41,7 +41,7 @@ import ( "time" "github.com/fsnotify/fsnotify" - "github.com/shirou/gopsutil/v3/process" + "github.com/shirou/gopsutil/v4/process" "infini.sh/framework/core/task" "infini.sh/framework/core/wrapper/taskset" "infini.sh/framework/modules/configs/client" diff --git a/core/host/sys_info.go b/core/host/sys_info.go index 13317ffe3..f59b533c8 100644 --- a/core/host/sys_info.go +++ b/core/host/sys_info.go @@ -26,11 +26,11 @@ package host import ( "fmt" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/disk" - "github.com/shirou/gopsutil/v3/host" - "github.com/shirou/gopsutil/v3/mem" - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/disk" + "github.com/shirou/gopsutil/v4/host" + "github.com/shirou/gopsutil/v4/mem" + "github.com/shirou/gopsutil/v4/net" "infini.sh/framework/core/errors" "runtime" "time" diff --git a/go.mod b/go.mod index 5a4ee6646..2b6827f25 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/andybalholm/brotli v1.1.1 github.com/arl/statsviz v0.6.0 github.com/bkaradzic/go-lz4 v1.0.0 - github.com/buger/jsonparser v1.1.1 + github.com/buger/jsonparser v1.1.2 github.com/caddyserver/certmagic v0.25.3 github.com/cihub/seelog v0.0.0-00010101000000-000000000000 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc @@ -20,7 +20,6 @@ require ( github.com/fsnotify/fsnotify v1.9.0 github.com/go-ldap/ldap/v3 v3.4.13 github.com/go-redis/redis/v8 v8.11.5 - github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang-jwt/jwt/v4 v4.5.2 github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f github.com/google/go-cmp v0.7.0 @@ -34,6 +33,7 @@ require ( github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/kardianos/service v1.2.2 github.com/klauspost/compress v1.18.0 + github.com/libdns/libdns v1.1.1 github.com/magiconair/properties v1.8.10 github.com/mailru/easyjson v0.9.0 github.com/minio/minio-go/v7 v7.0.90 @@ -46,11 +46,10 @@ require ( github.com/ryanuber/go-glob v1.0.0 github.com/savsgio/gotils v0.0.0-20250408102913-196191ec6287 github.com/segmentio/encoding v0.4.1 - github.com/seiflotfy/cuckoofilter v0.0.0-20240715131351-a2f2c23f1771 github.com/shaj13/go-guardian/v2 v2.11.6 - github.com/shirou/gopsutil/v3 v3.24.5 + github.com/shirou/gopsutil/v4 v4.26.3 github.com/spf13/viper v1.20.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/twmb/franz-go v1.18.1 github.com/twmb/franz-go/pkg/kadm v1.16.0 github.com/twmb/franz-go/pkg/kmsg v1.11.2 @@ -65,6 +64,7 @@ require ( golang.org/x/time v0.11.0 golang.org/x/tools v0.44.0 google.golang.org/grpc v1.71.1 + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/hjson/hjson-go.v3 v3.3.0 gopkg.in/square/go-jose.v2 v2.6.0 @@ -80,9 +80,9 @@ require ( github.com/caddyserver/zerossl v0.1.5 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgraph-io/ristretto/v2 v2.2.0 // indirect - github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.0 // indirect github.com/fatih/color v1.18.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect @@ -93,7 +93,6 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -105,7 +104,6 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/libdns/libdns v1.1.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -119,19 +117,18 @@ require ( github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/segmentio/asm v1.1.3 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.12.0 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tklauser/go-sysconf v0.3.12 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/x448/float16 v0.8.4 // indirect @@ -147,9 +144,7 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/term v0.42.0 // indirect google.golang.org/appengine v1.6.6 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect diff --git a/go.sum b/go.sum index 3d3f64898..e1f48ebba 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +code.pfad.fr/check v1.1.0 h1:GWvjdzhSEgHvEHe2uJujDcpmZoySKuHQNrZMfzfO0bE= +code.pfad.fr/check v1.1.0/go.mod h1:NiUH13DtYsb7xp5wll0U4SXx7KhXQVCtRgdC96IPfoM= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= -github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -12,9 +12,8 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= -github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= -github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI= +github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/arl/statsviz v0.6.0 h1:jbW1QJkEYQkufd//4NDYRSNBpwJNrdzPahF7ZmoGdyE= @@ -24,8 +23,8 @@ github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6 github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= -github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/caddyserver/certmagic v0.25.3 h1:mGf5ba8F7xA4c5jfDZZbK2buY1VEkbnwpMDixaju94A= github.com/caddyserver/certmagic v0.25.3/go.mod h1:YVs43D5+H/Dckt4bTga1KSO/xYfFBfVZainGDywYPAA= github.com/caddyserver/zerossl v0.1.5 h1:dkvOjBAEEtY6LIGAHei7sw2UgqSD6TrWweXpV7lvEvE= @@ -44,13 +43,13 @@ github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINA github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 h1:BS21ZUJ/B5X2UVUbczfmdWH7GapPWAhxcMsDnjJTU1E= -github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -73,9 +72,9 @@ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-ldap/ldap/v3 v3.2.4/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg= -github.com/go-ldap/ldap/v3 v3.4.11 h1:4k0Yxweg+a3OyBLjdYn5OKglv18JNvfDykSoI8bW0gU= -github.com/go-ldap/ldap/v3 v3.4.11/go.mod h1:bY7t0FLK8OAVpp/vV6sSlpz3EQDGcQwc8pF0ujLgKvM= github.com/go-ldap/ldap/v3 v3.4.13 h1:+x1nG9h+MZN7h/lUi5Q3UZ0fJ1GyDQYbPvbuH38baDQ= github.com/go-ldap/ldap/v3 v3.4.13/go.mod h1:LxsGZV6vbaK0sIvYfsv47rfh4ca0JXokCoKjZxsszv0= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -100,8 +99,6 @@ github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PU github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f h1:16RtHeWGkJMc80Etb8RPCcKevXGldr57+LOyZt8zOlg= @@ -192,8 +189,6 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -204,6 +199,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/letsencrypt/challtestsrv v1.4.2 h1:0ON3ldMhZyWlfVNYYpFuWRTmZNnyfiL9Hh5YzC3JVwU= +github.com/letsencrypt/challtestsrv v1.4.2/go.mod h1:GhqMqcSoeGpYd5zX5TgwA6er/1MbWzx/o7yuuVya+Wk= +github.com/letsencrypt/pebble/v2 v2.10.0 h1:Wq6gYXlsY6ubqI3hhxsTzdyotvfdjFBxuwYqCLCnj/U= +github.com/letsencrypt/pebble/v2 v2.10.0/go.mod h1:Sk8cmUIPcIdv2nINo+9PB4L+ZBhzY+F9A1a/h/xmWiQ= github.com/libdns/libdns v1.1.1 h1:wPrHrXILoSHKWJKGd0EiAVmiJbFShguILTg9leS/P/U= github.com/libdns/libdns v1.1.1/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -223,12 +222,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mholt/acmez/v3 v3.1.2 h1:auob8J/0FhmdClQicvJvuDavgd5ezwLBfKuYmynhYzc= -github.com/mholt/acmez/v3 v3.1.2/go.mod h1:L1wOU06KKvq7tswuMDwKdcHeKpFFgkppZy/y0DFxagQ= github.com/mholt/acmez/v3 v3.1.6 h1:eGVQNObP0pBN4sxqrXeg7MYqTOWyoiYpQqITVWlrevk= github.com/mholt/acmez/v3 v3.1.6/go.mod h1:5nTPosTGosLxF3+LU4ygbgMRFDhbAVpqMI4+a4aHLBY= -github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= -github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= @@ -275,8 +270,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/r3labs/diff/v2 v2.15.1 h1:EOrVqPUzi+njlumoqJwiS/TgGgmZo83619FNDB9xQUg= github.com/r3labs/diff/v2 v2.15.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= @@ -297,17 +292,11 @@ github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.4.1 h1:KLGaLSW0jrmhB58Nn4+98spfvPvmo4Ci1P/WIQ9wn7w= github.com/segmentio/encoding v0.4.1/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= -github.com/seiflotfy/cuckoofilter v0.0.0-20240715131351-a2f2c23f1771 h1:emzAzMZ1L9iaKCTxdy3Em8Wv4ChIAGnfiz18Cda70g4= -github.com/seiflotfy/cuckoofilter v0.0.0-20240715131351-a2f2c23f1771/go.mod h1:bR6DqgcAl1zTcOX8/pE2Qkj9XO00eCNqmKb7lXP8EAg= github.com/shaj13/go-guardian/v2 v2.11.6 h1:N0UgnL+AI0IH59eii0H0QnQEesyPPmGFB1h9g1MkZ8g= github.com/shaj13/go-guardian/v2 v2.11.6/go.mod h1:rSe5VLuWu9EyUT68Xi6qxb/DJc+ajiqPAq+VKhEUKkE= github.com/shaj13/libcache v1.0.0/go.mod h1:YCq92Zosqj4erhlLdm2Mu1cX2FDAxjfFOxTphzN7S9U= -github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= -github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -333,14 +322,14 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= -github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= -github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/twmb/franz-go v1.18.1 h1:D75xxCDyvTqBSiImFx2lkPduE39jz1vaD7+FNc+vMkc= github.com/twmb/franz-go v1.18.1/go.mod h1:Uzo77TarcLTUZeLuGq+9lNpSkfZI+JErv7YJhlDjs9M= github.com/twmb/franz-go/pkg/kadm v1.16.0 h1:STMs1t5lYR5mR974PSiwNzE5TvsosByTp+rKXLOhAjE= @@ -375,18 +364,12 @@ go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= @@ -395,14 +378,10 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -413,8 +392,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -425,8 +402,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -439,22 +414,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -466,8 +433,6 @@ golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= -golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -479,8 +444,6 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= @@ -495,8 +458,6 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= -gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= -gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/hjson/hjson-go.v3 v3.3.0 h1:F/aKL7cJ3rTfjQIxdetssl+ryKBz0V1mLJnrBs6ljFg= gopkg.in/hjson/hjson-go.v3 v3.3.0/go.mod h1:X6zrTSVeImfwfZLfgQdInl9mWjqPqgH90jom9nym/lw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -511,7 +472,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.18.8/go.mod h1:d/CXqwWv+Z2XEG1LgceeDmHQwpUJhROPx16SlxJgERY= diff --git a/lib/status/fs.go b/lib/status/fs.go index 95910343a..ee0d4370f 100644 --- a/lib/status/fs.go +++ b/lib/status/fs.go @@ -4,7 +4,7 @@ package status import ( log "github.com/cihub/seelog" - disk2 "github.com/shirou/gopsutil/v3/disk" + disk2 "github.com/shirou/gopsutil/v4/disk" ) type DiskStatus struct { diff --git a/modules/metrics/host/cpu/cpu.go b/modules/metrics/host/cpu/cpu.go index acd3f69ad..4e2ff1c24 100644 --- a/modules/metrics/host/cpu/cpu.go +++ b/modules/metrics/host/cpu/cpu.go @@ -31,8 +31,8 @@ import ( "strconv" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/cpu" - "github.com/shirou/gopsutil/v3/load" + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/load" "infini.sh/framework/core/config" "infini.sh/framework/core/event" "infini.sh/framework/core/util" diff --git a/modules/metrics/host/disk/disk.go b/modules/metrics/host/disk/disk.go index b994fba10..28f2d7fbf 100644 --- a/modules/metrics/host/disk/disk.go +++ b/modules/metrics/host/disk/disk.go @@ -34,7 +34,7 @@ import ( "strings" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/disk" + "github.com/shirou/gopsutil/v4/disk" "infini.sh/framework/core/config" "infini.sh/framework/core/event" "infini.sh/framework/core/util" diff --git a/modules/metrics/host/memory/memory.go b/modules/metrics/host/memory/memory.go index 53ea9a9cf..1b8cb0413 100644 --- a/modules/metrics/host/memory/memory.go +++ b/modules/metrics/host/memory/memory.go @@ -32,7 +32,7 @@ import ( "strings" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/mem" + "github.com/shirou/gopsutil/v4/mem" "infini.sh/framework/core/config" "infini.sh/framework/core/errors" "infini.sh/framework/core/event" diff --git a/modules/metrics/host/network/network.go b/modules/metrics/host/network/network.go index e867753dc..fda0bc965 100644 --- a/modules/metrics/host/network/network.go +++ b/modules/metrics/host/network/network.go @@ -31,7 +31,7 @@ import ( "syscall" log "github.com/cihub/seelog" - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/net" "infini.sh/framework/core/config" "infini.sh/framework/core/errors" "infini.sh/framework/core/event" diff --git a/modules/metrics/host/network/sockstat_linux.go b/modules/metrics/host/network/sockstat_linux.go index ccdf3c3c6..f64047004 100644 --- a/modules/metrics/host/network/sockstat_linux.go +++ b/modules/metrics/host/network/sockstat_linux.go @@ -47,7 +47,7 @@ import ( "bufio" "fmt" "github.com/pkg/errors" - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/net" "infini.sh/framework/core/util" "os" ) diff --git a/modules/metrics/host/network/sockstat_other.go b/modules/metrics/host/network/sockstat_other.go index b9c878407..cce297723 100644 --- a/modules/metrics/host/network/sockstat_other.go +++ b/modules/metrics/host/network/sockstat_other.go @@ -44,7 +44,7 @@ package network import ( - "github.com/shirou/gopsutil/v3/net" + "github.com/shirou/gopsutil/v4/net" "infini.sh/framework/core/util" ) diff --git a/modules/metrics/host/overall/netspeed_darwin.go b/modules/metrics/host/overall/netspeed_darwin.go new file mode 100644 index 000000000..482038cf2 --- /dev/null +++ b/modules/metrics/host/overall/netspeed_darwin.go @@ -0,0 +1,89 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build darwin + +package overall + +import ( + "os/exec" + "strings" + + log "github.com/cihub/seelog" + "github.com/shirou/gopsutil/v4/net" +) + +// detectNetworkBandwidthPerInterface detects network interface speeds in Mbps for each interface. +// On macOS, it uses ifconfig to get network interface speeds. +// Returns a map of interface name to bandwidth in Mbps. +func detectNetworkBandwidthPerInterface() map[string]float64 { + result := make(map[string]float64) + + interfaces, err := net.IOCounters(true) + if err != nil { + log.Debugf("overall: failed to get network interfaces: %v", err) + return result + } + + for _, iface := range interfaces { + // Skip loopback and virtual interfaces + if isVirtualInterface(iface.Name) { + continue + } + + speed := detectDarwinInterfaceSpeed(iface.Name) + if speed > 0 { + log.Debugf("overall: detected interface %s speed: %.0f Mbps", iface.Name, speed) + result[iface.Name] = speed + } + } + + return result +} + +// detectDarwinInterfaceSpeed attempts to detect the speed of a single interface on macOS +func detectDarwinInterfaceSpeed(ifaceName string) float64 { + // Try ifconfig for link speed + out, err := exec.Command("ifconfig", ifaceName).Output() + if err != nil { + return 0 + } + + output := string(out) + + // Look for "media: autoselect (1000baseT )" + if strings.Contains(output, "10Gbase") || strings.Contains(output, "10GBASE") { + return 10000 + } + if strings.Contains(output, "1000baseT") || strings.Contains(output, "1000BASE-T") { + return 1000 + } + if strings.Contains(output, "100baseT") || strings.Contains(output, "100BASE-T") { + return 100 + } + if strings.Contains(output, "10baseT") || strings.Contains(output, "10BASE-T") { + return 10 + } + + return 0 +} diff --git a/modules/metrics/host/overall/netspeed_linux.go b/modules/metrics/host/overall/netspeed_linux.go new file mode 100644 index 000000000..ea5649d16 --- /dev/null +++ b/modules/metrics/host/overall/netspeed_linux.go @@ -0,0 +1,75 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build linux + +package overall + +import ( + "fmt" + "os" + "strconv" + "strings" + + log "github.com/cihub/seelog" + "github.com/shirou/gopsutil/v4/net" +) + +// detectNetworkBandwidthPerInterface detects network interface speeds in Mbps for each interface. +// On Linux, it reads from /sys/class/net//speed for each interface. +// Returns a map of interface name to bandwidth in Mbps. +func detectNetworkBandwidthPerInterface() map[string]float64 { + result := make(map[string]float64) + + interfaces, err := net.IOCounters(true) + if err != nil { + log.Debugf("overall: failed to get network interfaces: %v", err) + return result + } + + for _, iface := range interfaces { + // Skip loopback and virtual interfaces + if isVirtualInterface(iface.Name) { + continue + } + + speedPath := fmt.Sprintf("/sys/class/net/%s/speed", iface.Name) + data, err := os.ReadFile(speedPath) + if err != nil { + log.Debugf("overall: failed to read speed for %s: %v", iface.Name, err) + continue + } + + speedStr := strings.TrimSpace(string(data)) + speed, err := strconv.ParseFloat(speedStr, 64) + if err != nil || speed <= 0 { + // Speed might be -1 if link is down or unknown + continue + } + + log.Debugf("overall: detected interface %s speed: %.0f Mbps", iface.Name, speed) + result[iface.Name] = speed + } + + return result +} diff --git a/modules/metrics/host/overall/netspeed_windows.go b/modules/metrics/host/overall/netspeed_windows.go new file mode 100644 index 000000000..4b8bcb2b6 --- /dev/null +++ b/modules/metrics/host/overall/netspeed_windows.go @@ -0,0 +1,142 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build windows + +package overall + +import ( + "os/exec" + "regexp" + "strconv" + "strings" + + log "github.com/cihub/seelog" +) + +// detectNetworkBandwidthPerInterface detects network interface speeds in Mbps for each interface. +// On Windows, it uses PowerShell/WMI to query network adapter speeds. +// Returns a map of interface name to bandwidth in Mbps. +func detectNetworkBandwidthPerInterface() map[string]float64 { + result := make(map[string]float64) + + // Use PowerShell to get network adapter speeds with names + cmd := exec.Command("powershell", "-Command", + "Get-NetAdapter | Where-Object {$_.Status -eq 'Up'} | Select-Object Name,LinkSpeed | ForEach-Object { $_.Name + '|' + $_.LinkSpeed }") + out, err := cmd.Output() + if err != nil { + log.Debugf("overall: failed to get network adapter speed via PowerShell: %v", err) + return tryWMICPerInterface() + } + + lines := strings.Split(string(out), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + parts := strings.SplitN(line, "|", 2) + if len(parts) != 2 { + continue + } + + name := strings.TrimSpace(parts[0]) + linkSpeed := strings.TrimSpace(parts[1]) + speed := parseWindowsLinkSpeed(linkSpeed) + if speed > 0 { + log.Debugf("overall: detected interface %s speed: %.0f Mbps", name, speed) + result[name] = speed + } + } + + return result +} + +// tryWMICPerInterface tries to get network speed using wmic (fallback for older Windows) +func tryWMICPerInterface() map[string]float64 { + result := make(map[string]float64) + + cmd := exec.Command("wmic", "nic", "where", "NetEnabled=true", "get", "Name,Speed") + out, err := cmd.Output() + if err != nil { + log.Debugf("overall: failed to get network adapter speed via wmic: %v", err) + return result + } + + lines := strings.Split(string(out), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "Name") { + continue + } + + // WMIC output is space-separated, speed is the last field + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + + speedStr := fields[len(fields)-1] + name := strings.Join(fields[:len(fields)-1], " ") + + // Speed from wmic is in bits per second + bps, err := strconv.ParseFloat(speedStr, 64) + if err != nil || bps <= 0 { + continue + } + mbps := bps / 1000000.0 + log.Debugf("overall: detected interface %s speed: %.0f Mbps", name, mbps) + result[name] = mbps + } + + return result +} + +// parseWindowsLinkSpeed parses Windows link speed strings like "1 Gbps", "100 Mbps" +func parseWindowsLinkSpeed(linkSpeed string) float64 { + linkSpeed = strings.TrimSpace(linkSpeed) + + // Match patterns like "1 Gbps", "100 Mbps", "10 Gbps" + gbpsRegex := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*[Gg]bps`) + mbpsRegex := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*[Mm]bps`) + kbpsRegex := regexp.MustCompile(`(\d+(?:\.\d+)?)\s*[Kk]bps`) + + if matches := gbpsRegex.FindStringSubmatch(linkSpeed); len(matches) > 1 { + if speed, err := strconv.ParseFloat(matches[1], 64); err == nil { + return speed * 1000 // Convert Gbps to Mbps + } + } + if matches := mbpsRegex.FindStringSubmatch(linkSpeed); len(matches) > 1 { + if speed, err := strconv.ParseFloat(matches[1], 64); err == nil { + return speed + } + } + if matches := kbpsRegex.FindStringSubmatch(linkSpeed); len(matches) > 1 { + if speed, err := strconv.ParseFloat(matches[1], 64); err == nil { + return speed / 1000 // Convert Kbps to Mbps + } + } + + return 0 +} diff --git a/modules/metrics/host/overall/overall.go b/modules/metrics/host/overall/overall.go new file mode 100644 index 000000000..81fed49c0 --- /dev/null +++ b/modules/metrics/host/overall/overall.go @@ -0,0 +1,636 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package overall + +import ( + "runtime" + "strings" + "sync" + + log "github.com/cihub/seelog" + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/disk" + "github.com/shirou/gopsutil/v4/mem" + "github.com/shirou/gopsutil/v4/net" + "infini.sh/framework/core/config" + "infini.sh/framework/core/event" + "infini.sh/framework/core/util" +) + +// DefaultBandwidthMbps is the default network bandwidth in Mbps when auto-detection fails +const DefaultBandwidthMbps = 1000 + +// Metric collects overall system utilization percentages for CPU, memory, disk, disk I/O and network. +// Each disk and network interface is monitored independently to identify specific bottlenecks. +type Metric struct { + Enabled bool `config:"enabled"` + IntervalSeconds float64 `config:"interval_seconds"` + YellowThreshold float64 `config:"yellow_threshold"` + RedThreshold float64 `config:"red_threshold"` + + mu sync.Mutex + + // Per-disk I/O tracking: map[deviceName] -> snapshot + prevDiskIO map[string]*diskIOSnapshot + + // Per-network interface tracking: map[ifaceName] -> snapshot + prevNetIO map[string]*netIOSnapshot + + // Per-network interface bandwidth (auto-detected): map[ifaceName] -> Mbps + netBandwidth map[string]float64 + + // Previous aggregate CPU times, used to compute the steal-time percentage + // between collections. nil before the first sample is taken. + prevCPUTimes *cpu.TimesStat + + // Previous TCP RetransSegs / OutSegs counters, used to compute both the + // retransmits-per-second rate and the retransmit ratio (retrans/out, a + // capacity-like 0-100% signal). hasPrevTCPRetrans guards against emitting + // bogus values on the first call (when the counters are unknown) and + // after a counter reset. + prevTCPRetrans int64 + prevTCPOutSegs int64 + hasPrevTCPRetrans bool +} + +// diskIOSnapshot stores previous I/O counters for a disk device. +// weightedIO is gopsutil's WeightedIO field (in milliseconds), used to derive +// the average queue depth (iostat "aqu-sz") across the collection interval. +type diskIOSnapshot struct { + readTime uint64 + writeTime uint64 + weightedIO uint64 +} + +// netIOSnapshot stores previous I/O counters for a network interface +type netIOSnapshot struct { + bytesRecv uint64 + bytesSent uint64 +} + +// deviceUtilization represents utilization info for a single device. +// queueDepth is only populated for disk I/O devices (avg outstanding requests +// over the collection interval, derived from WeightedIO). +type deviceUtilization struct { + name string + usedPercent float64 + queueDepth float64 +} + +func New(cfg *config.Config) (*Metric, error) { + me := &Metric{ + Enabled: true, + IntervalSeconds: 10, + YellowThreshold: 70, + RedThreshold: 90, + prevDiskIO: make(map[string]*diskIOSnapshot), + prevNetIO: make(map[string]*netIOSnapshot), + netBandwidth: make(map[string]float64), + } + + err := cfg.Unpack(&me) + if err != nil { + panic(err) + } + + // Initialize network bandwidth detection for all interfaces + me.initNetworkBandwidth() + + log.Debugf("overall utilization metric enabled") + return me, nil +} + +// initNetworkBandwidth detects and stores bandwidth for each network interface +func (m *Metric) initNetworkBandwidth() { + bandwidths := detectNetworkBandwidthPerInterface() + for name, bw := range bandwidths { + m.netBandwidth[name] = bw + log.Debugf("overall: interface %s bandwidth: %.0f Mbps", name, bw) + } +} + +// Collect gathers CPU, memory, disk, disk I/O and network utilization +// and emits a "host/overall" event with raw values for the front layer to interpret. +// Each disk and network interface is monitored independently. +func (m *Metric) Collect() error { + if !m.Enabled { + return nil + } + + fields := util.MapStr{} + + // Collect all metrics + cpuPercent, cpuStealPercent := m.collectCPU() + memPercent := m.collectMemory() + diskPercent, diskInodePercent := m.collectDiskUsage() + diskIODevices := m.collectDiskIO() + netDevices := m.collectNetwork() + tcpRetransPerSec, tcpRetransPercent, hasTCPRetrans := m.collectTCPRetrans() + + // --- CPU utilization --- + fields["cpu.used_percent"] = cpuPercent + fields["cpu.steal_percent"] = cpuStealPercent + + // --- Memory utilization --- + fields["memory.used_percent"] = memPercent + + // --- Disk capacity utilization --- + fields["disk.used_percent"] = diskPercent + fields["disk.inodes_used_percent"] = diskInodePercent + + // --- Per-disk I/O utilization --- + diskIOMap := util.MapStr{} + var maxDiskIO deviceUtilization + for _, dev := range diskIODevices { + diskIOMap[dev.name] = util.MapStr{ + "used_percent": dev.usedPercent, + "queue_depth": dev.queueDepth, + } + if dev.usedPercent > maxDiskIO.usedPercent { + maxDiskIO = dev + } + } + if len(diskIOMap) > 0 { + fields["disk_io.devices"] = diskIOMap + fields["disk_io.used_percent"] = maxDiskIO.usedPercent + fields["disk_io.queue_depth"] = maxDiskIO.queueDepth + fields["disk_io.bottleneck_device"] = maxDiskIO.name + } + + // --- Per-network interface utilization --- + netMap := util.MapStr{} + var maxNet deviceUtilization + for _, dev := range netDevices { + bw := m.netBandwidth[dev.name] + if bw <= 0 { + bw = DefaultBandwidthMbps // Default if unknown + } + netMap[dev.name] = util.MapStr{ + "used_percent": dev.usedPercent, + "bandwidth_mbps": bw, + } + if dev.usedPercent > maxNet.usedPercent { + maxNet = dev + } + } + if len(netMap) > 0 { + fields["network.devices"] = netMap + fields["network.used_percent"] = maxNet.usedPercent + fields["network.bottleneck_device"] = maxNet.name + } + + // --- TCP retransmits --- + // tcp_retrans_per_sec is a raw throughput-style signal. tcp_retrans_percent + // is the retransmit ratio (RetransSegs/OutSegs over the interval), a true + // 0-100% capacity-like signal that is directly comparable to the other + // utilization percentages and therefore folded into status/bottleneck + // below as part of the network subsystem. Only emit once we have a valid + // delta (skips first call and counter resets). + if hasTCPRetrans { + fields["network.tcp_retrans_per_sec"] = tcpRetransPerSec + fields["network.tcp_retrans_percent"] = tcpRetransPercent + } + + // --- Calculate overall status and bottleneck --- + // CPU stress can come from either user/system load or hypervisor steal; + // disk pressure can come from either capacity or inode exhaustion; network + // pressure can come from either link saturation or a high TCP retransmit + // ratio. Pick the worst signal in each subsystem so the bottleneck + // reflects reality. + cpuStatus := cpuPercent + if cpuStealPercent > cpuStatus { + cpuStatus = cpuStealPercent + } + diskStatus := diskPercent + if diskInodePercent > diskStatus { + diskStatus = diskInodePercent + } + netStatus := maxNet + if hasTCPRetrans && tcpRetransPercent > netStatus.usedPercent { + // Surface as bottleneck="network:tcp_retrans" via existing naming. + netStatus = deviceUtilization{name: "tcp_retrans", usedPercent: tcpRetransPercent} + } + status, bottleneck := m.calculateStatus(cpuStatus, memPercent, diskStatus, maxDiskIO, netStatus) + fields["status"] = status + fields["bottleneck"] = bottleneck + + return event.Save(&event.Event{ + Metadata: event.EventMetadata{ + Category: "host", + Name: "overall", + Datatype: "gauge", + }, + Fields: util.MapStr{ + "host": util.MapStr{ + "overall": fields, + }, + }, + }) +} + +// collectCPU returns the current overall CPU utilization percentage and the +// hypervisor steal-time percentage (both 0-100). The steal percentage is +// derived from the delta of cpu.Times() across collections; it is reported as +// 0 on the first call (no baseline) and on platforms where Steal is unavailable. +func (m *Metric) collectCPU() (usedPercent, stealPercent float64) { + percents, err := cpu.Percent(0, false) + if err != nil { + log.Errorf("overall: failed to get cpu percent: %v", err) + } + if len(percents) > 0 { + usedPercent = percents[0] + } + + times, err := cpu.Times(false) + if err != nil || len(times) == 0 { + if err != nil { + log.Debugf("overall: failed to get cpu times: %v", err) + } + return usedPercent, 0 + } + cur := times[0] + + m.mu.Lock() + prev := m.prevCPUTimes + m.prevCPUTimes = &cur + m.mu.Unlock() + + if prev == nil { + // First sample; no delta available yet. + return usedPercent, 0 + } + + deltaTotal := cur.Total() - prev.Total() + deltaSteal := cur.Steal - prev.Steal + if deltaTotal <= 0 || deltaSteal < 0 { + // Counter reset or non-monotonic reading; skip this sample. + return usedPercent, 0 + } + stealPercent = deltaSteal / deltaTotal * 100.0 + if stealPercent > 100.0 { + stealPercent = 100.0 + } + return usedPercent, stealPercent +} + +// collectMemory returns the current memory utilization percentage (0-100). +func (m *Metric) collectMemory() float64 { + v, err := mem.VirtualMemory() + if err != nil { + log.Errorf("overall: failed to get memory info: %v", err) + return 0 + } + if v == nil { + return 0 + } + return v.UsedPercent +} + +// collectDiskUsage returns disk capacity and inode utilization percentages +// (both 0-100). Inode usage is aggregated across all partitions (sum of used +// inodes over sum of total inodes); filesystems without inodes (e.g. some +// pseudo-filesystems, certain Windows volumes) are skipped. +func (m *Metric) collectDiskUsage() (usedPercent, inodesUsedPercent float64) { + if runtime.GOOS == "darwin" { + v, err := disk.Usage("/") + if err != nil { + log.Errorf("overall: failed to get disk usage: %v", err) + return 0, 0 + } + return v.UsedPercent, v.InodesUsedPercent + } + + partitions, err := disk.Partitions(false) + if err != nil || len(partitions) == 0 { + log.Errorf("overall: failed to get disk partitions: %v", err) + return 0, 0 + } + var total, used, inodesTotal, inodesUsed uint64 + for _, p := range partitions { + if p.Device == "" { + continue + } + v, err := disk.Usage(p.Mountpoint) + if err != nil { + continue + } + total += v.Total + used += v.Used + inodesTotal += v.InodesTotal + inodesUsed += v.InodesUsed + } + if total > 0 { + usedPercent = float64(used) / float64(total) * 100.0 + } + if inodesTotal > 0 { + inodesUsedPercent = float64(inodesUsed) / float64(inodesTotal) * 100.0 + } + return usedPercent, inodesUsedPercent +} + +// collectDiskIO returns per-disk I/O utilization percentages (0-100) based on io time deltas. +// Returns empty slice if data is not yet available (first call). +func (m *Metric) collectDiskIO() []deviceUtilization { + ret, err := disk.IOCounters() + if err != nil { + log.Debugf("overall: failed to get disk io counters: %v", err) + return nil + } + if len(ret) == 0 { + return nil + } + + m.mu.Lock() + defer m.mu.Unlock() + + var results []deviceUtilization + + for name, io := range ret { + // Skip certain device types + if strings.HasPrefix(name, "loop") || strings.HasPrefix(name, "ram") { + continue + } + + prev, exists := m.prevDiskIO[name] + if !exists { + // First time seeing this device, store initial values + m.prevDiskIO[name] = &diskIOSnapshot{ + readTime: io.ReadTime, + writeTime: io.WriteTime, + weightedIO: io.WeightedIO, + } + continue + } + + // Calculate IO busy time delta + deltaRead := io.ReadTime - prev.readTime + deltaWrite := io.WriteTime - prev.writeTime + deltaIO := deltaRead + deltaWrite + deltaWeighted := io.WeightedIO - prev.weightedIO + + // Update stored values + prev.readTime = io.ReadTime + prev.writeTime = io.WriteTime + prev.weightedIO = io.WeightedIO + + // IO busy time delta in ms over the collection interval + intervalMs := m.IntervalSeconds * 1000.0 + busy := float64(deltaIO) / intervalMs * 100.0 + if busy > 100.0 { + busy = 100.0 + } + if busy < 0 { + busy = 0 + } + + // Average queue depth over the interval (iostat "aqu-sz"): + // WeightedIO is the cumulative weighted time spent doing I/Os in ms, + // so dividing the delta by the interval in ms yields the average + // number of in-flight requests. WeightedIO is always 0 on platforms + // that don't populate it (e.g. macOS), which correctly yields 0. + var queueDepth float64 + if intervalMs > 0 { + queueDepth = float64(deltaWeighted) / intervalMs + } + if queueDepth < 0 { + queueDepth = 0 + } + + results = append(results, deviceUtilization{ + name: name, + usedPercent: busy, + queueDepth: queueDepth, + }) + } + + return results +} + +// collectTCPRetrans returns the TCP retransmits-per-second rate and the +// retransmit ratio (RetransSegs/OutSegs over the interval, 0-100%) computed +// from the delta of the kernel's TCP counters. The boolean return is false +// on the first call (no baseline), on platforms where ProtoCounters is not +// supported (currently non-Linux), or when the counters are unavailable / have +// been reset (negative delta). +func (m *Metric) collectTCPRetrans() (perSec, percent float64, ok bool) { + if m.IntervalSeconds <= 0 { + return 0, 0, false + } + counters, err := net.ProtoCounters([]string{"tcp"}) + if err != nil || len(counters) == 0 { + // Platforms without ProtoCounters support (darwin, windows, ...) end + // up here; log at debug level to avoid spamming production logs. + if err != nil { + log.Debugf("overall: failed to get tcp proto counters: %v", err) + } + return 0, 0, false + } + stats := counters[0].Stats + curRetrans, hasRetrans := stats["RetransSegs"] + curOut, hasOut := stats["OutSegs"] + if !hasRetrans || !hasOut { + return 0, 0, false + } + + m.mu.Lock() + prevRetrans := m.prevTCPRetrans + prevOut := m.prevTCPOutSegs + hadPrev := m.hasPrevTCPRetrans + m.prevTCPRetrans = curRetrans + m.prevTCPOutSegs = curOut + m.hasPrevTCPRetrans = true + m.mu.Unlock() + + if !hadPrev { + return 0, 0, false + } + deltaRetrans := curRetrans - prevRetrans + deltaOut := curOut - prevOut + if deltaRetrans < 0 || deltaOut < 0 { + // Counter reset (e.g. kernel restart, namespace change); skip. + return 0, 0, false + } + perSec = float64(deltaRetrans) / m.IntervalSeconds + if deltaOut > 0 { + percent = float64(deltaRetrans) / float64(deltaOut) * 100.0 + if percent > 100.0 { + percent = 100.0 + } + } + return perSec, percent, true +} + +// collectNetwork returns per-interface network utilization percentages (0-100) +// based on throughput relative to each interface's detected bandwidth. +// Returns empty slice if data is not yet available (first call). +func (m *Metric) collectNetwork() []deviceUtilization { + stats, err := net.IOCounters(true) // true = per-interface + if err != nil { + log.Debugf("overall: failed to get network io counters: %v", err) + return nil + } + + m.mu.Lock() + defer m.mu.Unlock() + + var results []deviceUtilization + + for _, stat := range stats { + name := stat.Name + + // Skip loopback and virtual interfaces + if isVirtualInterface(name) { + continue + } + + prev, exists := m.prevNetIO[name] + if !exists { + // First time seeing this interface, store initial values + m.prevNetIO[name] = &netIOSnapshot{ + bytesRecv: stat.BytesRecv, + bytesSent: stat.BytesSent, + } + continue + } + + // Calculate deltas + deltaRecv := stat.BytesRecv - prev.bytesRecv + deltaSent := stat.BytesSent - prev.bytesSent + + // Update stored values + prev.bytesRecv = stat.BytesRecv + prev.bytesSent = stat.BytesSent + + // Use the higher of in/out throughput for utilization + deltaMax := deltaRecv + if deltaSent > deltaMax { + deltaMax = deltaSent + } + + // Get bandwidth for this interface + bandwidth := m.netBandwidth[name] + if bandwidth <= 0 { + bandwidth = DefaultBandwidthMbps // Default if unknown + } + + // Convert bandwidth from Mbps to bytes/sec: Mbps * 1_000_000 / 8 + bandwidthBytesPerSec := bandwidth * 1000000.0 / 8.0 + + throughputBytesPerSec := float64(deltaMax) / m.IntervalSeconds + percent := throughputBytesPerSec / bandwidthBytesPerSec * 100.0 + if percent > 100.0 { + percent = 100.0 + } + if percent < 0 { + percent = 0 + } + + results = append(results, deviceUtilization{ + name: name, + usedPercent: percent, + }) + } + + return results +} + +// isVirtualInterface returns true if the interface name looks like a virtual/loopback interface +func isVirtualInterface(name string) bool { + // Common virtual interface prefixes across platforms + virtualPrefixes := []string{ + "lo", "lo0", // Loopback + "veth", "docker", "br-", // Docker/containers + "virbr", "vnet", // Libvirt/KVM + "utun", "awdl", "bridge", "llw", "ap", "XHC", // macOS virtual + "vmnet", // VMware + "Loopback", // Windows loopback + } + + for _, prefix := range virtualPrefixes { + if strings.HasPrefix(name, prefix) || name == prefix { + return true + } + } + return false +} + +// calculateStatus determines the overall system status (green/yellow/red) and +// identifies the bottleneck subsystem (if any) based on configured thresholds. +// For disk_io and network, it includes the specific device name in the bottleneck. +func (m *Metric) calculateStatus(cpuPct, memPct, diskPct float64, maxDiskIO, maxNet deviceUtilization) (status, bottleneck string) { + status = "green" + bottleneck = "" + + // Subsystems to check with their utilization percentages + type subsystem struct { + name string + percent float64 + device string // Optional device name for disk_io and network + } + + subsystems := []subsystem{ + {"cpu", cpuPct, ""}, + {"memory", memPct, ""}, + {"disk", diskPct, ""}, + } + + // Add disk_io if we have data + if maxDiskIO.name != "" { + subsystems = append(subsystems, subsystem{"disk_io", maxDiskIO.usedPercent, maxDiskIO.name}) + } + + // Add network if we have data + if maxNet.name != "" { + subsystems = append(subsystems, subsystem{"network", maxNet.usedPercent, maxNet.name}) + } + + // Find the highest utilization and determine status + var maxPercent float64 + var maxSubsystem subsystem + for _, s := range subsystems { + if s.percent > maxPercent { + maxPercent = s.percent + maxSubsystem = s + } + } + + // Determine status based on thresholds + if maxPercent >= m.RedThreshold { + status = "red" + if maxSubsystem.device != "" { + bottleneck = maxSubsystem.name + ":" + maxSubsystem.device + } else { + bottleneck = maxSubsystem.name + } + } else if maxPercent >= m.YellowThreshold { + status = "yellow" + if maxSubsystem.device != "" { + bottleneck = maxSubsystem.name + ":" + maxSubsystem.device + } else { + bottleneck = maxSubsystem.name + } + } + + return status, bottleneck +} diff --git a/modules/metrics/metrics.go b/modules/metrics/metrics.go index 148f4e556..ebfbd2d6f 100755 --- a/modules/metrics/metrics.go +++ b/modules/metrics/metrics.go @@ -39,6 +39,7 @@ import ( "infini.sh/framework/modules/metrics/host/disk" "infini.sh/framework/modules/metrics/host/memory" "infini.sh/framework/modules/metrics/host/network" + "infini.sh/framework/modules/metrics/host/overall" agent2 "infini.sh/framework/modules/metrics/instance" ) @@ -55,6 +56,7 @@ type MetricConfig struct { DiskConfig *Config `config:"disk"` CPUConfig *Config `config:"cpu"` MemoryConfig *Config `config:"memory"` + OverallConfig *Config `config:"overall"` ElasticsearchConfig *Config `config:"elasticsearch"` Tags []string `config:"tags"` @@ -275,6 +277,28 @@ func (module *MetricsModule) CollectHostMetric() { } task.RegisterScheduleTask(memTask) } + + if module.config.OverallConfig != nil { + overallM, err := overall.New(module.config.OverallConfig) + if err != nil { + panic(err) + } + if overallM.Enabled { + taskId := util.GetUUID() + module.taskIDs = append(module.taskIDs, taskId) + var overallTask = task.ScheduleTask{ + ID: taskId, + Description: "fetch overall utilization metrics", + Type: "interval", + Interval: "10s", + Task: func(ctx context.Context) { + log.Debug("collecting overall utilization metrics") + overallM.Collect() + }, + } + task.RegisterScheduleTask(overallTask) + } + } } func (module *MetricsModule) Start() error { diff --git a/modules/stats/simple.go b/modules/stats/simple.go index c02dcc19d..3ee895883 100755 --- a/modules/stats/simple.go +++ b/modules/stats/simple.go @@ -40,7 +40,7 @@ import ( log "github.com/cihub/seelog" "github.com/segmentio/encoding/json" - "github.com/shirou/gopsutil/v3/process" + "github.com/shirou/gopsutil/v4/process" "infini.sh/framework/core/api" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/env" From 4f38ae25abeb8f0f2d0f352cc6e84e2d91312f4e Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 27 May 2026 11:59:59 +0800 Subject: [PATCH 064/137] fix(pipeline): add task type aliases for branch compatibility Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- modules/pipeline/model.go | 2 ++ modules/pipeline/proto.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/modules/pipeline/model.go b/modules/pipeline/model.go index bdbf305d7..0fce2368b 100644 --- a/modules/pipeline/model.go +++ b/modules/pipeline/model.go @@ -42,6 +42,8 @@ type PipelineStatus struct { Processors []map[string]interface{} `json:"processor"` } +type PipelineTaskStatus = PipelineStatus + type PipelineResult struct { Success bool `json:"success"` Error string `json:"error,omitempty"` diff --git a/modules/pipeline/proto.go b/modules/pipeline/proto.go index cfabbb9d0..f3ecf7516 100644 --- a/modules/pipeline/proto.go +++ b/modules/pipeline/proto.go @@ -27,6 +27,8 @@ import "infini.sh/framework/core/pipeline" type GetPipelinesResponse map[string]*PipelineStatus +type GetPipelineTasksResponse = GetPipelinesResponse + type CreatePipelineRequest struct { pipeline.PipelineConfigV2 Processors []map[string]interface{} `json:"processor"` From 78891af815ef262b5c65f2cdd33a0f61c0146a65 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 27 May 2026 21:58:30 +0800 Subject: [PATCH 065/137] fix: cluster register with metrics collect --- modules/elastic/metadata.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index 1c6cff6ce..24e75c33c 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -177,6 +177,41 @@ func updateClusterHealthStatus(clusterID string, healthStatus string) { } +func SyncClusterHealthStatus(clusterID string) { + if strings.TrimSpace(clusterID) == "" { + return + } + + metadata := elastic.GetMetadata(clusterID) + if metadata == nil || metadata.Config == nil { + return + } + if metadata.Config.Source != elastic.ElasticsearchConfigSourceElasticsearch { + return + } + + healthStatus := "unavailable" + if metadata.IsAvailable() { + if client := elastic.GetClientNoPanic(clusterID); client != nil { + health, err := client.ClusterHealth(nil) + if err == nil && health != nil && health.StatusCode == 200 && strings.TrimSpace(health.Status) != "" { + metadata.Health = health + healthStatus = health.Status + } else if metadata.Health != nil && strings.TrimSpace(metadata.Health.Status) != "" { + healthStatus = metadata.Health.Status + } else { + healthStatus = "green" + } + } else if metadata.Health != nil && strings.TrimSpace(metadata.Health.Status) != "" { + healthStatus = metadata.Health.Status + } else { + healthStatus = "green" + } + } + + updateClusterHealthStatus(clusterID, healthStatus) +} + // update cluster state, on state version change func (module *ElasticModule) updateClusterState(clusterId string, force bool) { From 8143a077f00bbe3a88169c3f3ee20b5915a25160 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 28 May 2026 14:49:51 +0800 Subject: [PATCH 066/137] fix: console behind nginx with tls --- core/api/client.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/api/client.go b/core/api/client.go index 6a9140d5f..56a84225a 100755 --- a/core/api/client.go +++ b/core/api/client.go @@ -66,8 +66,10 @@ func SimpleGetTLSConfig(tlsConfig *config.TLSConfig) *tls.Config { } func GetClientTLSConfig(tlsConfig *config.TLSConfig) (*tls.Config, error) { - - pool := x509.NewCertPool() + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } skipVerify := tlsConfig.TLSInsecureSkipVerify if tlsConfig.TLSBypassMalformedCert { From 123be0bf0fbfcdde0d98497f2604888deefa4aa8 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 28 May 2026 15:47:58 +0800 Subject: [PATCH 067/137] fix: logs search and check polling --- modules/elastic/metadata.go | 73 ++++++++++++++++++++----------------- modules/elastic/module.go | 2 +- 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index 24e75c33c..5685c5da7 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -53,44 +53,49 @@ func (module *ElasticModule) clusterHealthCheck(clusterID string, force bool) { log.Tracef("execute health check for: %v", clusterID) cfg := elastic.GetConfig(clusterID) + if cfg == nil || !cfg.Enabled { + return + } + if !force && !cfg.Monitored { + log.Tracef("skip health check for unmonitored cluster: %v", clusterID) + return + } metadata := elastic.GetOrInitMetadata(cfg) - if cfg.Enabled || force { - //check seeds' availability - if force { - //add seeds to host for health check - hosts := metadata.GetSeedHosts() - for _, host := range hosts { - elastic.GetOrInitHost(host, clusterID) - } + //check seeds' availability + if force { + //add seeds to host for health check + hosts := metadata.GetSeedHosts() + for _, host := range hosts { + elastic.GetOrInitHost(host, clusterID) } - //metadata.GetHttpClient(metadata.GetActivePreferredSeedEndpoint()) - client := elastic.GetClient(cfg.ID) - //check cluster health status - health, err := client.ClusterHealth(nil) - if err != nil || health == nil || health.StatusCode != 200 { - if health != nil && util.ContainStr(util.UnsafeBytesToString(health.RawResult.Body), "master_not_discovered_exception") { - metadata.ReportFailure(errors.New("master_not_discovered_exception")) - } else { - metadata.ReportFailure(err) - } - if metadata.Config.Source == elastic.ElasticsearchConfigSourceElasticsearch && !metadata.IsAvailable() { - updateClusterHealthStatus(clusterID, "unavailable") - } + } + //metadata.GetHttpClient(metadata.GetActivePreferredSeedEndpoint()) + client := elastic.GetClient(cfg.ID) + //check cluster health status + health, err := client.ClusterHealth(nil) + if err != nil || health == nil || health.StatusCode != 200 { + if health != nil && util.ContainStr(util.UnsafeBytesToString(health.RawResult.Body), "master_not_discovered_exception") { + metadata.ReportFailure(errors.New("master_not_discovered_exception")) } else { - if metadata.Health == nil || metadata.Health.NumberOfNodes == 0 || metadata.Health.Status != health.Status || !metadata.IsAvailable() || force { - if metadata.Config.Source == elastic.ElasticsearchConfigSourceElasticsearch { - updateClusterHealthStatus(clusterID, health.Status) - } - log.Tracef("cluster [%v] health [%v] updated", clusterID, metadata.Health) - } - changes, err := util.DiffTwoObject(metadata.Health, health) - if err != nil { - log.Errorf("diff cluster health error: %v", err) - } - metadata.ReportSuccess() - if len(changes) > 0 { - metadata.Health = health + metadata.ReportFailure(err) + } + if metadata.Config.Source == elastic.ElasticsearchConfigSourceElasticsearch && !metadata.IsAvailable() { + updateClusterHealthStatus(clusterID, "unavailable") + } + } else { + if metadata.Health == nil || metadata.Health.NumberOfNodes == 0 || metadata.Health.Status != health.Status || !metadata.IsAvailable() || force { + if metadata.Config.Source == elastic.ElasticsearchConfigSourceElasticsearch { + updateClusterHealthStatus(clusterID, health.Status) } + log.Tracef("cluster [%v] health [%v] updated", clusterID, metadata.Health) + } + changes, err := util.DiffTwoObject(metadata.Health, health) + if err != nil { + log.Errorf("diff cluster health error: %v", err) + } + metadata.ReportSuccess() + if len(changes) > 0 { + metadata.Health = health } } } diff --git a/modules/elastic/module.go b/modules/elastic/module.go index cd7b8e733..bfe48dea5 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -509,7 +509,7 @@ func (module *ElasticModule) Start() error { } cfg1, ok := value.(*elastic.ElasticsearchConfig) if ok && cfg1 != nil { - if !cfg1.Enabled || (cfg1.MetadataConfigs != nil && !cfg1.MetadataConfigs.HealthCheck.Enabled) { + if !cfg1.Enabled || !cfg1.Monitored || (cfg1.MetadataConfigs != nil && !cfg1.MetadataConfigs.HealthCheck.Enabled) { return true } From a59bb4ac813aa6185a218c0a1902e5488ad00135 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 28 May 2026 16:44:18 +0800 Subject: [PATCH 068/137] fix: monitor disable and cluster delete for clean kv --- core/kv/kv.go | 11 +++++++ modules/elastic/metadata.go | 31 +++++++++++++++--- modules/elastic/module.go | 40 +++++++++++++----------- modules/metrics/elastic/elasticsearch.go | 6 +++- plugins/badger/badger.go | 16 ++++++++-- plugins/simple_kv/simple.go | 11 +++++++ 6 files changed, 87 insertions(+), 28 deletions(-) diff --git a/core/kv/kv.go b/core/kv/kv.go index 56e7662c8..fc0695884 100755 --- a/core/kv/kv.go +++ b/core/kv/kv.go @@ -30,6 +30,7 @@ package kv import ( log "github.com/cihub/seelog" "infini.sh/framework/core/errors" + "time" ) type KVStore interface { @@ -42,8 +43,10 @@ type KVStore interface { GetCompressedValue(bucket string, key []byte) ([]byte, error) AddValueCompress(bucket string, key []byte, value []byte) error + AddValueCompressWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error AddValue(bucket string, key []byte, value []byte) error + AddValueWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error ExistsKey(bucket string, key []byte) (bool, error) @@ -82,10 +85,18 @@ func AddValueCompress(bucket string, key []byte, value []byte) error { return getKVHandler().AddValueCompress(bucket, key, value) } +func AddValueCompressWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error { + return getKVHandler().AddValueCompressWithTTL(bucket, key, value, ttl) +} + func AddValue(bucket string, key []byte, value []byte) error { return getKVHandler().AddValue(bucket, key, value) } +func AddValueWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error { + return getKVHandler().AddValueWithTTL(bucket, key, value, ttl) +} + func ExistsKey(bucket string, key []byte) (bool, error) { return getKVHandler().ExistsKey(bucket, key) } diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index 5685c5da7..0bcb6bdf6 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -48,6 +48,8 @@ import ( "infini.sh/framework/core/util" ) +const elasticMetadataKVRetention = 30 * 24 * time.Hour + func (module *ElasticModule) clusterHealthCheck(clusterID string, force bool) { log.Tracef("execute health check for: %v", clusterID) @@ -224,6 +226,9 @@ func (module *ElasticModule) updateClusterState(clusterId string, force bool) { if meta == nil { return } + if !force && !meta.Config.Monitored { + return + } if !force && !meta.IsAvailable() { return @@ -261,7 +266,7 @@ func (module *ElasticModule) updateClusterState(clusterId string, force bool) { if err != nil { log.Errorf("failed to load index metadata from es: %v", err) } - err = kv.AddValueCompress(elastic.KVElasticIndexMetadata, []byte(clusterId), oldIndexState) + err = kv.AddValueCompressWithTTL(elastic.KVElasticIndexMetadata, []byte(clusterId), oldIndexState, elasticMetadataKVRetention) if err != nil { log.Errorf("failed to save index metadata: %v", err) } @@ -724,7 +729,7 @@ func (module *ElasticModule) saveIndexMetadata(state *elastic.ClusterState, clus } if isIndicesStateChange { - err = kv.AddValueCompress(elastic.KVElasticIndexMetadata, []byte(clusterID), util.MustToJSONBytes(newIndexMetadata)) + err = kv.AddValueCompressWithTTL(elastic.KVElasticIndexMetadata, []byte(clusterID), util.MustToJSONBytes(newIndexMetadata), elasticMetadataKVRetention) if err != nil { log.Error(err) } @@ -736,6 +741,10 @@ func (module *ElasticModule) updateNodeInfo(meta *elastic.ElasticsearchMetadata, log.Trace("update node info") + if !force && !meta.Config.Monitored { + return + } + if !force && !meta.IsAvailable() { stateChanged := false if !force { @@ -838,7 +847,7 @@ func (module *ElasticModule) updateNodeInfo(meta *elastic.ElasticsearchMetadata, "nodes": nodes, "timestamp": time.Now(), } - err = kv.AddValueCompress(elastic.KVElasticNodeMetadata, []byte(meta.Config.ID), util.MustToJSONBytes(cacheNodeInfo)) + err = kv.AddValueCompressWithTTL(elastic.KVElasticNodeMetadata, []byte(meta.Config.ID), util.MustToJSONBytes(cacheNodeInfo), elasticMetadataKVRetention) if err != nil { log.Errorf("save node metadata error: %v", err) } @@ -1188,6 +1197,9 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro // on demand, on state version change func updateAliases(meta *elastic.ElasticsearchMetadata, force bool) { + if !force && !meta.Config.Monitored { + return + } if !force && !meta.IsAvailable() { return @@ -1288,6 +1300,9 @@ func (module *ElasticModule) updateClusterSettings(clusterId string) { if meta == nil { return } + if !meta.Config.Monitored { + return + } if !meta.IsAvailable() { return } @@ -1362,9 +1377,15 @@ func (module *ElasticModule) updateClusterSettings(clusterId string) { if err != nil { panic(err) } - kv.AddValue(elastic.KVElasticClusterSettings, []byte(clusterId), util.MustToJSONBytes(settings)) + err = kv.AddValueWithTTL(elastic.KVElasticClusterSettings, []byte(clusterId), util.MustToJSONBytes(settings), elasticMetadataKVRetention) + if err != nil { + log.Errorf("failed to save cluster settings: %v", err) + } } else { - kv.AddValue(elastic.KVElasticClusterSettings, []byte(clusterId), util.MustToJSONBytes(settings)) + err = kv.AddValueWithTTL(elastic.KVElasticClusterSettings, []byte(clusterId), util.MustToJSONBytes(settings), elasticMetadataKVRetention) + if err != nil { + log.Errorf("failed to save cluster settings: %v", err) + } } } diff --git a/modules/elastic/module.go b/modules/elastic/module.go index bfe48dea5..cf20510be 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -258,7 +258,7 @@ func nodeAvailabilityCheck() { } cfg := elastic.GetConfig(v.ClusterID) - if !cfg.Enabled || (cfg.MetadataConfigs != nil && !cfg.MetadataConfigs.NodeAvailabilityCheck.Enabled) { + if !cfg.Enabled || !cfg.Monitored || (cfg.MetadataConfigs != nil && !cfg.MetadataConfigs.NodeAvailabilityCheck.Enabled) { return true } @@ -325,7 +325,7 @@ func (module *ElasticModule) registerClusterStateRefreshTask() { log.Tracef("init meta refresh task: [%v] [%v] [%v] [%v]", key, v.ID, v.Name, v.Enabled) if ok { - if !v.Enabled || (v.MetadataConfigs != nil && !v.MetadataConfigs.MetadataRefresh.Enabled) { + if !v.Enabled || !v.Monitored || (v.MetadataConfigs != nil && !v.MetadataConfigs.MetadataRefresh.Enabled) { return true } @@ -475,23 +475,25 @@ func (module *ElasticModule) Start() error { cfg1, ok := value.(*elastic.ElasticsearchConfig) if ok && cfg1 != nil { log.Tracef("init elasticsearch config: %v", cfg1.Name) - metadata := elastic.GetMetadata(cfg1.ID) - if metadata != nil { - //update nodes - module.updateNodeInfo(metadata, true, cfg1.Discovery.Enabled) + if cfg1.Monitored { + metadata := elastic.GetMetadata(cfg1.ID) + if metadata != nil { + //update nodes + module.updateNodeInfo(metadata, true, cfg1.Discovery.Enabled) - //update alias - updateAliases(metadata, true) + //update alias + updateAliases(metadata, true) - //update - module.updateClusterState(cfg1.ID, true) - } + //update + module.updateClusterState(cfg1.ID, true) + } - task.RunWithContext("cluster_health_check", func(ctx context.Context) error { - id := task.MustGetString(ctx, "id") - module.clusterHealthCheck(id, true) - return nil - }, context.WithValue(context.Background(), "id", cfg1.ID)) + task.RunWithContext("cluster_health_check", func(ctx context.Context) error { + id := task.MustGetString(ctx, "id") + module.clusterHealthCheck(id, true) + return nil + }, context.WithValue(context.Background(), "id", cfg1.ID)) + } } return true }) @@ -679,7 +681,7 @@ func (module *ElasticModule) registerClusterSettingsRefreshTask() { log.Tracef("init settings refresh task: [%v] [%v] [%v] [%v]", key, v.ID, v.Name, v.Enabled) if ok { - if !v.Enabled || (v.MetadataConfigs != nil && !v.MetadataConfigs.ClusterSettingsCheck.Enabled) { + if !v.Enabled || !v.Monitored || (v.MetadataConfigs != nil && !v.MetadataConfigs.ClusterSettingsCheck.Enabled) { return true } if startTime, ok := module.settingsMap.Load(v.ID); ok { @@ -735,7 +737,7 @@ func (module *ElasticModule) refreshAllClusterMetadata() { return true } v.Config = cfg - if !cfg.Enabled || (cfg.MetadataConfigs != nil && !cfg.MetadataConfigs.MetadataRefresh.Enabled) { + if !cfg.Enabled || !cfg.Monitored || (cfg.MetadataConfigs != nil && !cfg.MetadataConfigs.MetadataRefresh.Enabled) { return true } module.updateNodeInfo(v, false, cfg.Discovery.Enabled) @@ -759,7 +761,7 @@ func (module *ElasticModule) refreshAllClusterAlias(force bool) { return true } v.Config = cfg - if !cfg.Enabled || (cfg.MetadataConfigs != nil && !cfg.MetadataConfigs.MetadataRefresh.Enabled) { + if !cfg.Enabled || !cfg.Monitored || (cfg.MetadataConfigs != nil && !cfg.MetadataConfigs.MetadataRefresh.Enabled) { return true } updateAliases(v, force) diff --git a/modules/metrics/elastic/elasticsearch.go b/modules/metrics/elastic/elasticsearch.go index 70d170775..ac8d3bfa5 100644 --- a/modules/metrics/elastic/elasticsearch.go +++ b/modules/metrics/elastic/elasticsearch.go @@ -259,7 +259,11 @@ func (m *ElasticsearchMetric) InitialCollectTask(k string, v *elastic.Elasticsea } } if !m.shouldCollectMetrics(v) { - log.Debugf("cluster [%v] NOT eligible for metrics collection (enabled[%v], monitored[%v], mode[%v], available[%v]), skip collect", v.Config.Name, v.Config.Enabled, v.Config.Monitored, v.Config.MetricCollectionMode, v.IsAvailable()) + available := false + if v.Config.Enabled && v.Config.Monitored { + available = v.IsAvailable() + } + log.Debugf("cluster [%v] NOT eligible for metrics collection (enabled[%v], monitored[%v], mode[%v], available[%v]), skip collect", v.Config.Name, v.Config.Enabled, v.Config.Monitored, v.Config.MetricCollectionMode, available) return true } if global.Env().IsDebug { diff --git a/plugins/badger/badger.go b/plugins/badger/badger.go index f18eaaf38..05d4752f5 100644 --- a/plugins/badger/badger.go +++ b/plugins/badger/badger.go @@ -283,6 +283,10 @@ func (filter *Module) GetCompressedValue(bucket string, key []byte) ([]byte, err } func (filter *Module) AddValueCompress(bucket string, key []byte, value []byte) error { + return filter.AddValueCompressWithTTL(bucket, key, value, 0) +} + +func (filter *Module) AddValueCompressWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error { value, err := lz4.Encode(nil, value) if err != nil { log.Error("Failed to encode:", err) @@ -291,7 +295,7 @@ func (filter *Module) AddValueCompress(bucket string, key []byte, value []byte) stats.Increment("badger", bucket+"::add_compress") - return filter.AddValue(bucket, key, value) + return filter.AddValueWithTTL(bucket, key, value, ttl) } func joinKey(bucket string, key []byte) []byte { @@ -299,6 +303,10 @@ func joinKey(bucket string, key []byte) []byte { } func (filter *Module) AddValue(bucket string, key []byte, value []byte) error { + return filter.AddValueWithTTL(bucket, key, value, 0) +} + +func (filter *Module) AddValueWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error { if filter.closed { return errors.New("module closed") } @@ -310,8 +318,10 @@ func (filter *Module) AddValue(bucket string, key []byte, value []byte) error { } bkt := filter.getOrInitBucket(bucket) err := bkt.Update(func(txn *badger.Txn) error { - err := txn.Set(key, value) - return err + if ttl > 0 { + return txn.SetEntry(badger.NewEntry(key, value).WithTTL(ttl)) + } + return txn.Set(key, value) }) return err } diff --git a/plugins/simple_kv/simple.go b/plugins/simple_kv/simple.go index f5510bf75..d5e7aad1b 100644 --- a/plugins/simple_kv/simple.go +++ b/plugins/simple_kv/simple.go @@ -30,6 +30,7 @@ package simple_kv import ( "errors" "sync" + "time" "github.com/bkaradzic/go-lz4" log "github.com/cihub/seelog" @@ -108,6 +109,11 @@ func (filter *SimpleKV) GetCompressedValue(bucket string, key []byte) ([]byte, e } func (filter *SimpleKV) AddValueCompress(bucket string, key []byte, value []byte) error { + return filter.AddValueCompressWithTTL(bucket, key, value, 0) +} + +func (filter *SimpleKV) AddValueCompressWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error { + _ = ttl value, err := lz4.Encode(nil, value) if err != nil { log.Error("Failed to encode:", err) @@ -122,6 +128,11 @@ func joinKey(bucket string, key []byte) string { } func (filter *SimpleKV) AddValue(bucket string, key []byte, value []byte) error { + return filter.AddValueWithTTL(bucket, key, value, 0) +} + +func (filter *SimpleKV) AddValueWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error { + _ = ttl if filter.closed { return errors.New("module closed") } From 198b577a1192faab1d59ca899c59f02f8c2816be Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 28 May 2026 16:51:34 +0800 Subject: [PATCH 069/137] fix: build error with impl --- modules/elastic/store.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/elastic/store.go b/modules/elastic/store.go index ae49f3c31..9a43c6f99 100755 --- a/modules/elastic/store.go +++ b/modules/elastic/store.go @@ -38,6 +38,7 @@ import ( "infini.sh/framework/core/util" "infini.sh/framework/modules/elastic/common" "net/http" + "time" ) type ElasticStore struct { @@ -122,12 +123,16 @@ func (store *ElasticStore) GetValue(bucket string, key []byte) ([]byte, error) { } func (store *ElasticStore) AddValueCompress(bucket string, key []byte, value []byte) error { + return store.AddValueCompressWithTTL(bucket, key, value, 0) +} + +func (store *ElasticStore) AddValueCompressWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error { value, err := lz4.Encode(nil, value) if err != nil { log.Error("Failed to encode:", bucket, ",", key, ",", err) return err } - return store.AddValue(bucket, key, value) + return store.AddValueWithTTL(bucket, key, value, ttl) } func getKey(bucket, key string) string { @@ -135,6 +140,11 @@ func getKey(bucket, key string) string { } func (store *ElasticStore) AddValue(bucket string, key []byte, value []byte) error { + return store.AddValueWithTTL(bucket, key, value, 0) +} + +func (store *ElasticStore) AddValueWithTTL(bucket string, key []byte, value []byte, ttl time.Duration) error { + _ = ttl file := Blob{} file.Content = base64.URLEncoding.EncodeToString(value) _, err := store.Client.Index(store.Config.IndexName, "_doc", getKey(bucket, string(key)), file, "") From cbf3390e374b8a1eca62c9ce657849bb910b94d6 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 09:42:22 +0800 Subject: [PATCH 070/137] feat: add token exchange for access --- core/access_token/authentication.go | 133 ++++++++++++++++++++++++++ core/access_token/module.go | 28 ++++++ core/api/basic_auth.go | 48 ++++++++++ core/api/basic_auth_test.go | 67 +++++++++++++ core/api/protected_routes.go | 48 ++++++++++ core/api/security_guard.go | 26 +++++ core/api/security_guard_test.go | 75 +++++++++++++++ core/api/web.go | 27 +++++- core/api/web_test.go | 20 +++- core/config/system.go | 10 +- core/credential/credential.go | 28 +++++- core/credential/domain.go | 95 +++++++++++++++--- core/model/const.go | 29 ++++++ core/model/instance.go | 3 +- go.sum | 2 + modules/api/api.go | 3 + modules/configs/client/client.go | 122 +++++++++++++++++------ modules/configs/client/client_test.go | 77 +++++++++++++++ modules/configs/common/domain.go | 15 ++- modules/web/web.go | 3 + 20 files changed, 807 insertions(+), 52 deletions(-) create mode 100644 core/access_token/authentication.go create mode 100644 core/access_token/module.go create mode 100644 core/api/basic_auth_test.go create mode 100644 core/api/protected_routes.go create mode 100644 core/api/security_guard.go create mode 100644 core/api/security_guard_test.go create mode 100644 core/model/const.go create mode 100644 modules/configs/client/client_test.go diff --git a/core/access_token/authentication.go b/core/access_token/authentication.go new file mode 100644 index 000000000..a6e9a5236 --- /dev/null +++ b/core/access_token/authentication.go @@ -0,0 +1,133 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package access_token + +import ( + "net/http" + "strings" + + log "github.com/cihub/seelog" + "github.com/emirpasic/gods/sets/hashset" + "infini.sh/framework/core/credential" + "infini.sh/framework/core/errors" + "infini.sh/framework/core/model" + "infini.sh/framework/core/orm" + "infini.sh/framework/core/util" +) + +type AccessToken struct { + orm.ORMObjectBase + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Username string `json:"username,omitempty"` + Value string `json:"value,omitempty"` + Permissions []string `json:"permissions,omitempty"` +} + +func (a AccessToken) String() string { + return util.ToJson(a, false) +} + +func ValidatePermissionByAccessToken(req *http.Request) error { + token := strings.TrimSpace(req.Header.Get(model.API_TOKEN)) + if token == "" { + return nil + } + tokenObject, err := GetByToken(token) + if err != nil { + return errors.NewWithHTTPCode(http.StatusUnauthorized, "invalid access token") + } + reqTokenPermissions := req.URL.Query()["permission"] + if len(reqTokenPermissions) == 0 { + return nil + } + userPermissionsSet := hashset.New() + for _, item := range tokenObject.Permissions { + userPermissionsSet.Add(item) + } + for _, permission := range reqTokenPermissions { + if !userPermissionsSet.Contains(permission) { + return errors.NewWithHTTPCode(http.StatusUnauthorized, "invalid access token permissions") + } + } + return nil +} + +func GetByToken(token string) (*AccessToken, error) { + err, result := orm.GetBy("type", credential.AccessToken, credential.Credential{}) + if err != nil { + return nil, err + } + for _, item := range result.Result { + cred := credential.Credential{} + err := util.FromJSONBytes(util.MustToJSONBytes(item), &cred) + if err != nil { + return nil, err + } + payload, err := cred.DecodeAccessToken() + if err != nil { + return nil, err + } + if payload.Value.String() == token { + return &AccessToken{ + ORMObjectBase: cred.ORMObjectBase, + Name: cred.Name, + Description: payload.Description, + Username: payload.Username, + Value: payload.Value.String(), + Permissions: payload.Permissions, + }, nil + } + } + return nil, errors.NewWithHTTPCode(http.StatusNotFound, "access token not found") +} + +func AddPermissionFilterByAccessToken(base []string, req *http.Request) []string { + token := strings.TrimSpace(req.Header.Get(model.API_TOKEN)) + if token == "" { + return base + } + tokenObject, err := GetByToken(token) + if err != nil { + log.Error("error on get access token,", err) + return base + } + if len(tokenObject.Permissions) == 0 { + return base + } + set := hashset.New() + for _, item := range base { + set.Add(item) + } + for _, item := range tokenObject.Permissions { + set.Add(item) + } + values := make([]string, 0, set.Size()) + for _, item := range set.Values() { + if str, ok := item.(string); ok { + values = append(values, str) + } + } + return values +} diff --git a/core/access_token/module.go b/core/access_token/module.go new file mode 100644 index 000000000..1c74ab78d --- /dev/null +++ b/core/access_token/module.go @@ -0,0 +1,28 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package access_token + +func Init() error { + return nil +} diff --git a/core/api/basic_auth.go b/core/api/basic_auth.go index 934720378..763bb49ea 100644 --- a/core/api/basic_auth.go +++ b/core/api/basic_auth.go @@ -28,8 +28,13 @@ package api import ( + "crypto/subtle" httprouter "infini.sh/framework/core/api/router" "net/http" + "strings" + + "infini.sh/framework/core/model" + configcommon "infini.sh/framework/modules/configs/common" ) type BasicAuthFilter struct { @@ -37,9 +42,17 @@ type BasicAuthFilter struct { Password string } +var loadManagedAccessTokenFromKeystore = func() (string, error) { + return configcommon.LoadTokenFromKeystore(configcommon.AgentAccessTokenKeystoreKey) +} + // BasicAuth register api with basic auth func BasicAuth(h httprouter.Handle, requiredUser, requiredPassword string) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + if validateManagedAccessToken(r) { + h(w, r, ps) + return + } // Get the Basic Authentication credentials user, password, hasAuth := r.BasicAuth() @@ -60,6 +73,10 @@ func (filter *BasicAuthFilter) FilterHttpRouter(pattern string, h httprouter.Han func (filter *BasicAuthFilter) FilterHttpHandlerFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, request *http.Request) { + if validateManagedAccessToken(request) { + handler(w, request) + return + } // Get the Basic Authentication credentials user, password, hasAuth := request.BasicAuth() if hasAuth && user == filter.Username && password == filter.Password { @@ -72,3 +89,34 @@ func (filter *BasicAuthFilter) FilterHttpHandlerFunc(pattern string, handler fun http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) } } + +func validateManagedAccessToken(req *http.Request) bool { + tokenValue := ExtractBearerOrAPIToken(req) + if tokenValue == "" { + return false + } + expectedToken, err := loadManagedAccessTokenFromKeystore() + if err != nil || expectedToken == "" { + return false + } + return subtle.ConstantTimeCompare([]byte(expectedToken), []byte(tokenValue)) == 1 +} + +func ValidateManagedAccessTokenRequest(req *http.Request) bool { + return validateManagedAccessToken(req) +} + +func ExtractBearerOrAPIToken(req *http.Request) string { + if req == nil { + return "" + } + tokenValue := strings.TrimSpace(req.Header.Get(model.API_TOKEN)) + if tokenValue != "" { + return tokenValue + } + authHeader := strings.TrimSpace(req.Header.Get("Authorization")) + if len(authHeader) < len("Bearer ")+1 || !strings.EqualFold(authHeader[:len("Bearer ")], "Bearer ") { + return "" + } + return strings.TrimSpace(authHeader[len("Bearer "):]) +} diff --git a/core/api/basic_auth_test.go b/core/api/basic_auth_test.go new file mode 100644 index 000000000..d0912abbe --- /dev/null +++ b/core/api/basic_auth_test.go @@ -0,0 +1,67 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "testing" + + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/model" +) + +func TestBasicAuthAcceptsManagedAccessToken(t *testing.T) { + oldLoad := loadManagedAccessTokenFromKeystore + t.Cleanup(func() { + loadManagedAccessTokenFromKeystore = oldLoad + }) + loadManagedAccessTokenFromKeystore = func() (string, error) { + return "managed-token", nil + } + + handler := BasicAuth(func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + w.WriteHeader(http.StatusAccepted) + }, "api-user", "api-pass") + + for name, applyAuth := range map[string]func(*http.Request){ + "x-api-token": func(req *http.Request) { + req.Header.Set(model.API_TOKEN, "managed-token") + }, + "bearer-token": func(req *http.Request) { + req.Header.Set("Authorization", "Bearer managed-token") + }, + } { + t.Run(name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/stats", nil) + applyAuth(req) + recorder := httptest.NewRecorder() + handler(recorder, req, nil) + + if recorder.Code != http.StatusAccepted { + t.Fatalf("unexpected status: %d", recorder.Code) + } + }) + } +} + +func TestBasicAuthFallsBackToBasicAuthCredentials(t *testing.T) { + oldLoad := loadManagedAccessTokenFromKeystore + t.Cleanup(func() { + loadManagedAccessTokenFromKeystore = oldLoad + }) + loadManagedAccessTokenFromKeystore = func() (string, error) { + return "", nil + } + + handler := BasicAuth(func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + w.WriteHeader(http.StatusAccepted) + }, "api-user", "api-pass") + + req := httptest.NewRequest(http.MethodGet, "/stats", nil) + req.SetBasicAuth("api-user", "api-pass") + recorder := httptest.NewRecorder() + handler(recorder, req, nil) + + if recorder.Code != http.StatusAccepted { + t.Fatalf("unexpected status: %d", recorder.Code) + } +} diff --git a/core/api/protected_routes.go b/core/api/protected_routes.go new file mode 100644 index 000000000..7353469cf --- /dev/null +++ b/core/api/protected_routes.go @@ -0,0 +1,48 @@ +package api + +import httprouter "infini.sh/framework/core/api/router" + +type ProtectedAPIRoute struct { + Method Method + Path string +} + +var DefaultProtectedAPIRoutes = []ProtectedAPIRoute{ + {Method: GET, Path: "/stats"}, + {Method: GET, Path: "/queue/stats"}, + {Method: GET, Path: "/queue/:id/stats"}, + {Method: GET, Path: "/queue/:id/_scroll"}, + {Method: DELETE, Path: "/queue/:id"}, + {Method: DELETE, Path: "/queue/_search"}, + {Method: PUT, Path: "/queue/:id/consumer/:consumer_id/offset"}, + {Method: GET, Path: "/queue/:id/consumer/:consumer_id/offset"}, + {Method: DELETE, Path: "/queue/:id/consumer/:consumer_id"}, + {Method: DELETE, Path: "/queue/consumer/_search"}, + {Method: GET, Path: "/pipeline/tasks/"}, + {Method: POST, Path: "/pipeline/tasks/_search"}, + {Method: POST, Path: "/pipeline/task/:id/_start"}, + {Method: POST, Path: "/pipeline/task/:id/_stop"}, + {Method: GET, Path: "/pipeline/task/:id"}, + {Method: DELETE, Path: "/pipeline/task/:id"}, + {Method: GET, Path: "/config/"}, + {Method: PUT, Path: "/config/"}, + {Method: GET, Path: "/config/runtime"}, + {Method: GET, Path: "/setting/logger"}, + {Method: PUT, Path: "/setting/logger"}, + {Method: POST, Path: "/setting/logger"}, +} + +func RegisterProtectedUIRoutes(routes []ProtectedAPIRoute, handle httprouter.Handle, options ...Option) { + for _, route := range routes { + HandleUIMethod(route.Method, route.Path, handle, options...) + } +} + +func RegisterProtectedRouterRoutes(router *httprouter.Router, routes []ProtectedAPIRoute, handle httprouter.Handle) { + if router == nil { + return + } + for _, route := range routes { + router.Handle(string(route.Method), route.Path, handle) + } +} diff --git a/core/api/security_guard.go b/core/api/security_guard.go new file mode 100644 index 000000000..7cff00683 --- /dev/null +++ b/core/api/security_guard.go @@ -0,0 +1,26 @@ +package api + +import ( + "strings" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/errors" +) + +func ValidateServerExposureConfig(cfg *config.SystemConfig) error { + if cfg == nil { + return nil + } + if cfg.APIConfig.Enabled && !cfg.APIConfig.Security.Enabled { + return errors.Errorf("unsafe config: api.enabled requires api.security.enabled") + } + if cfg.WebAppConfig.Enabled && cfg.WebAppConfig.EmbeddingAPI { + return errors.Errorf("unsafe config: web.embedding_api is forbidden; use protected UI routes instead") + } + if cfg.APIConfig.Security.Enabled { + if strings.TrimSpace(cfg.APIConfig.Security.Username) == "" { + return errors.Errorf("unsafe config: api.security.username is required when api.security.enabled is true") + } + } + return nil +} diff --git a/core/api/security_guard_test.go b/core/api/security_guard_test.go new file mode 100644 index 000000000..610b01361 --- /dev/null +++ b/core/api/security_guard_test.go @@ -0,0 +1,75 @@ +package api + +import ( + "strings" + "testing" + + "infini.sh/framework/core/config" +) + +func TestValidateServerExposureConfig(t *testing.T) { + tests := []struct { + name string + cfg config.SystemConfig + wantErr string + }{ + { + name: "safe api config", + cfg: config.SystemConfig{ + APIConfig: config.APIConfig{ + Enabled: true, + Security: config.APISecurityConfig{ + Enabled: true, + Username: "api-user", + }, + }, + }, + }, + { + name: "reject insecure api", + cfg: config.SystemConfig{ + APIConfig: config.APIConfig{ + Enabled: true, + }, + }, + wantErr: "api.enabled requires api.security.enabled", + }, + { + name: "reject embedded api on web", + cfg: config.SystemConfig{ + WebAppConfig: config.WebAppConfig{ + Enabled: true, + EmbeddingAPI: true, + }, + }, + wantErr: "web.embedding_api is forbidden", + }, + { + name: "reject missing api username", + cfg: config.SystemConfig{ + APIConfig: config.APIConfig{ + Enabled: true, + Security: config.APISecurityConfig{ + Enabled: true, + }, + }, + }, + wantErr: "api.security.username is required", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateServerExposureConfig(&tt.cfg) + if tt.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) + } + }) + } +} diff --git a/core/api/web.go b/core/api/web.go index cd72cd121..fd6674cbe 100755 --- a/core/api/web.go +++ b/core/api/web.go @@ -119,7 +119,7 @@ func StartWeb(cfg config.WebAppConfig) { if registeredAPIMethodHandler != nil { for k, v := range registeredAPIMethodHandler { for m, n := range v { - if shouldSkipEmbeddedAPIRoute(m) { + if shouldSkipEmbeddedAPIRoute(k, m) { continue } log.Debug("register http handler: ", k, " ", m) @@ -129,7 +129,7 @@ func StartWeb(cfg config.WebAppConfig) { } if registeredAPIFuncHandler != nil { for k, v := range registeredAPIFuncHandler { - if shouldSkipEmbeddedAPIRoute(k) { + if shouldSkipEmbeddedAPIRoute("", k) { continue } log.Debug("register http handler: ", k) @@ -341,11 +341,28 @@ func shouldRegisterWebsocketOnWeb(cfg config.WebAppConfig) bool { return registeredAPIFuncHandler[getWebsocketRegistrationPath(cfg)] == nil } -func shouldSkipEmbeddedAPIRoute(path string) bool { - if registeredUIHandler == nil { +func shouldSkipEmbeddedAPIRoute(method, path string) bool { + if registeredUIHandler != nil { + if _, exists := registeredUIHandler[path]; exists { + return true + } + } + if registeredUIMethodHandler == nil { + return false + } + if method == "" { + for _, handlers := range registeredUIMethodHandler { + if _, exists := handlers[path]; exists { + return true + } + } + return false + } + methodHandlers, exists := registeredUIMethodHandler[Method(method)] + if !exists { return false } - _, exists := registeredUIHandler[path] + _, exists = methodHandlers[path] return exists } diff --git a/core/api/web_test.go b/core/api/web_test.go index 01e0949d9..0e0d8bf0e 100644 --- a/core/api/web_test.go +++ b/core/api/web_test.go @@ -4,6 +4,7 @@ import ( "net/http" "testing" + httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/config" ) @@ -54,18 +55,33 @@ func TestShouldRegisterWebsocketOnWeb(t *testing.T) { func TestShouldSkipEmbeddedAPIRoute(t *testing.T) { originalUIHandlers := registeredUIHandler + originalUIMethodHandlers := registeredUIMethodHandler t.Cleanup(func() { registeredUIHandler = originalUIHandlers + registeredUIMethodHandler = originalUIMethodHandlers }) registeredUIHandler = map[string]http.Handler{ "/": http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}), } + registeredUIMethodHandler = map[Method]map[string]RegisteredAPIHandler{ + GET: { + "/stats": { + Handler: func(http.ResponseWriter, *http.Request, httprouter.Params) {}, + }, + }, + } - if !shouldSkipEmbeddedAPIRoute("/") { + if !shouldSkipEmbeddedAPIRoute("", "/") { t.Fatal("expected API root route to be skipped when UI root is registered") } - if shouldSkipEmbeddedAPIRoute("/_info") { + if !shouldSkipEmbeddedAPIRoute(string(GET), "/stats") { + t.Fatal("expected method-based UI route to suppress embedded API registration") + } + if !shouldSkipEmbeddedAPIRoute("", "/stats") { + t.Fatal("expected UI method route to suppress embedded API func registration on same path") + } + if shouldSkipEmbeddedAPIRoute(string(GET), "/_info") { t.Fatal("expected unrelated API route not to be skipped") } } diff --git a/core/config/system.go b/core/config/system.go index 687860379..63fd640d3 100755 --- a/core/config/system.go +++ b/core/config/system.go @@ -289,8 +289,9 @@ type ConfigsConfig struct { ValidConfigsExtensions []string `config:"valid_config_extensions"` TLSConfig TLSConfig `config:"tls"` //server or client's certs ManagerConfig struct { - LocalConfigsRepoPath string `config:"local_configs_repo_path"` - BasicAuth BasicAuth `config:"basic_auth"` + LocalConfigsRepoPath string `config:"local_configs_repo_path"` + BasicAuth BasicAuth `config:"basic_auth"` + AccessToken ucfg.SecretString `config:"access_token"` } `config:"manager"` AlwaysRegisterAfterRestart bool `config:"always_register_after_restart"` AllowGeneratedMetricsTasks bool `config:"allow_generated_metrics_tasks"` @@ -330,10 +331,15 @@ type RealmConfig struct { type AuthenticationConfig struct { Native RealmConfig `config:"native"` + AccessToken AccessTokenConfig `config:"access_token"` HTTPBasicAuthProvider HTTPBasicAuthProvider `config:"http_basic"` OAuth map[string]OAuthConfig `config:"oauth"` } +type AccessTokenConfig struct { + Native RealmConfig `config:"native"` +} + type HTTPBasicAuthProvider struct { Enabled bool `config:"enabled"` Endpoint string `config:"endpoint" json:"endpoint,omitempty"` diff --git a/core/credential/credential.go b/core/credential/credential.go index c96970556..058a8379c 100644 --- a/core/credential/credential.go +++ b/core/credential/credential.go @@ -31,6 +31,7 @@ import ( "fmt" "infini.sh/framework/core/model" "infini.sh/framework/core/orm" + "infini.sh/framework/lib/go-ucfg" ) type Credential struct { @@ -71,6 +72,8 @@ func (cred *Credential) Encode() error { return encodeBasicAuth(cred) case Token: return encodeToken(cred) + case AccessToken: + return encodeAccessToken(cred) default: return fmt.Errorf("unkonow credential type [%s]", cred.Type) } @@ -100,18 +103,39 @@ func (cred *Credential) DecodeToken() (string, error) { return "", fmt.Errorf("unkonow credential type [%s]", cred.Type) } +func (cred *Credential) DecodeAccessToken() (*AccessTokenPayload, error) { + dv, err := cred.Decode() + if err != nil { + return nil, err + } + if token, ok := dv.(AccessTokenPayload); ok { + return &token, nil + } + return nil, fmt.Errorf("unkonow credential type [%s]", cred.Type) +} + func (cred *Credential) Decode() (interface{}, error) { switch cred.Type { case BasicAuth: return decodeBasicAuth(cred) case Token: return decodeToken(cred) + case AccessToken: + return decodeAccessToken(cred) default: return nil, fmt.Errorf("unkonow credential type [%s]", cred.Type) } } const ( - BasicAuth string = "basic_auth" - Token string = "token" + BasicAuth string = "basic_auth" + Token string = "token" + AccessToken string = "access_token" ) + +type AccessTokenPayload struct { + Value ucfg.SecretString `json:"value" yaml:"value"` + Permissions []string `json:"permissions,omitempty" yaml:"permissions,omitempty"` + Username string `json:"username,omitempty" yaml:"username,omitempty"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` +} diff --git a/core/credential/domain.go b/core/credential/domain.go index 63fcc99e4..7459ef032 100644 --- a/core/credential/domain.go +++ b/core/credential/domain.go @@ -158,13 +158,13 @@ func decodeBasicAuth(cred *Credential) (basicAuth model.BasicAuth, err error) { } func encodeToken(cred *Credential) error { - params, ok := cred.Payload[cred.Type].(map[string]interface{}) - if !ok { - return fmt.Errorf("wrong credential parameters for type [%s], expect a map", cred.Type) + params, err := getCredentialPayloadMap(cred) + if err != nil { + return err } - value, ok := params["value"].(string) - if !ok { - return fmt.Errorf("wrong credential parameters value for type [%s], expect a string", cred.Type) + value, err := getCredentialSecret(params) + if err != nil { + return err } if value == "" { return fmt.Errorf("credential parameters value can not be empty") @@ -188,14 +188,12 @@ func encodeToken(cred *Credential) error { } func decodeToken(cred *Credential) (token model.Token, err error) { - params, ok := cred.Payload[cred.Type].(map[string]interface{}) - if !ok { - err = fmt.Errorf("wrong credential parameters for type [%s], expect a map", cred.Type) + params, err := getCredentialPayloadMap(cred) + if err != nil { return } - value, ok := params["value"].(string) - if !ok { - err = fmt.Errorf("wrong credential parameters value for type [%s], expect a string", cred.Type) + value, err := getCredentialSecret(params) + if err != nil { return } if value == "" { @@ -222,6 +220,79 @@ func decodeToken(cred *Credential) (token model.Token, err error) { return } +func encodeAccessToken(cred *Credential) error { + return encodeToken(cred) +} + +func decodeAccessToken(cred *Credential) (payload AccessTokenPayload, err error) { + params, err := getCredentialPayloadMap(cred) + if err != nil { + return + } + value, err := getCredentialSecret(params) + if err != nil { + return + } + if value == "" { + err = fmt.Errorf("credential parameters value can not be empty") + return + } + salt, ok := cred.Encrypt.Params["salt"].(string) + if !ok { + err = fmt.Errorf("credential encrypt parameters salt can not be empty") + return + } + secret := cred.secret + if secret == nil { + secret, err = GetOrInitSecret() + if err != nil { + return payload, err + } + } + plaintext, err := util.AesGcmDecrypt([]byte(value), secret, []byte(salt)) + if err != nil { + return payload, err + } + payload.Value = ucfg.SecretString(plaintext) + if permissions, ok := params["permissions"].([]interface{}); ok { + for _, item := range permissions { + if str, ok := item.(string); ok { + payload.Permissions = append(payload.Permissions, str) + } + } + } else if permissions, ok := params["permissions"].([]string); ok { + payload.Permissions = append(payload.Permissions, permissions...) + } + if username, ok := params["username"].(string); ok { + payload.Username = username + } + if description, ok := params["description"].(string); ok { + payload.Description = description + } + return +} + +func getCredentialPayloadMap(cred *Credential) (map[string]interface{}, error) { + params, ok := cred.Payload[cred.Type].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("wrong credential parameters for type [%s], expect a map", cred.Type) + } + return params, nil +} + +func getCredentialSecret(params map[string]interface{}) (string, error) { + switch value := params["value"].(type) { + case string: + return value, nil + case []byte: + return string(value), nil + case ucfg.SecretString: + return string(value.Get()), nil + default: + return "", fmt.Errorf("wrong credential parameters value, expect a string") + } +} + type ChangeEvent func(credentials *Credential) var changeEvents []ChangeEvent diff --git a/core/model/const.go b/core/model/const.go new file mode 100644 index 000000000..7a9cdca63 --- /dev/null +++ b/core/model/const.go @@ -0,0 +1,29 @@ +// Copyright (C) INFINI Labs & INFINI LIMITED. +// +// The INFINI Framework is offered under the GNU Affero General Public License v3.0 +// and as commercial software. +// +// For commercial licensing, contact us at: +// - Website: infinilabs.com +// - Email: hello@infini.ltd +// +// Open Source licensed under AGPL V3: +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package model + +const ( + CredentialIDSystemKey = "credential_id" + API_TOKEN = "X-API-TOKEN" +) diff --git a/core/model/instance.go b/core/model/instance.go index c090bc5c4..e5064ed7a 100644 --- a/core/model/instance.go +++ b/core/model/instance.go @@ -55,7 +55,8 @@ type Instance struct { //application information Application env.Application `json:"application,omitempty" elastic_mapping:"application: { type: object }"` - BasicAuth *BasicAuth `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"` + BasicAuth *BasicAuth `config:"basic_auth" json:"basic_auth,omitempty" elastic_mapping:"basic_auth:{type:object}"` + AccessToken *Token `config:"access_token" json:"access_token,omitempty" elastic_mapping:"access_token:{type:object}"` ManagerCredentialID string `json:"manager_credential_id,omitempty" elastic_mapping:"manager_credential_id:{type:keyword}"` AccessCredentialID string `json:"access_credential_id,omitempty" elastic_mapping:"access_credential_id:{type:keyword}"` diff --git a/go.sum b/go.sum index e1f48ebba..ee8093407 100644 --- a/go.sum +++ b/go.sum @@ -445,6 +445,8 @@ google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuh google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20170918111702-1e559d0a00ee/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.2.1-0.20170921194603-d4b75ebd4f9f/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= diff --git a/modules/api/api.go b/modules/api/api.go index 67c7c01b3..613fe1626 100755 --- a/modules/api/api.go +++ b/modules/api/api.go @@ -187,6 +187,9 @@ func (module *APIModule) Setup() { } func (module *APIModule) Start() error { + if err := api.ValidateServerExposureConfig(global.Env().SystemConfig); err != nil { + return err + } api.StartAPI() return nil } diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index 2925c3b1a..009adb595 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -34,6 +34,7 @@ import ( "net/url" "os" "path/filepath" + "strings" "sync" "time" @@ -53,14 +54,19 @@ import ( const bucketName = "instance_registered" const configRegisterEnvKey = "CONFIG_MANAGED_SUCCESS" -func ConnectToManager() error { +var postRegisterHooks []func(server string, res *util.Result) error - if !global.Env().SystemConfig.Configs.Managed { +func ConnectToManager() error { + cfg := global.Env().SystemConfig.Configs + if !cfg.Managed { return nil } + if cfg.Servers == nil || len(cfg.Servers) == 0 { + return errors.Errorf("no config manager was found") + } // k8s env setting always_register_after_restart and pod after restart the ip will change so need register again - if !global.Env().SystemConfig.Configs.AlwaysRegisterAfterRestart { + if !cfg.AlwaysRegisterAfterRestart { if exists, err := kv.ExistsKey(bucketName, []byte(global.Env().SystemConfig.NodeConfig.ID)); exists && err == nil { //already registered skip further process log.Info("already registered to config manager") @@ -71,25 +77,16 @@ func ConnectToManager() error { log.Info("register new instance to config manager") - //register to config manager - if global.Env().SystemConfig.Configs.Servers == nil || len(global.Env().SystemConfig.Configs.Servers) == 0 { - return errors.Errorf("no config manager was found") - } - info := model.GetInstanceInfo() registerReq := common.InstanceRegisterRequest{ Client: info, } - if info.Application.Name == "agent" { - accessToken, err := common.EnsureTokenInKeystore(common.AgentAccessTokenKeystoreKey) - if err != nil { - return err - } - registerReq.AccessToken = &common.RegisterToken{ - Name: fmt.Sprintf("%s reverse access token", info.ID), - Description: fmt.Sprintf("Console to Agent access token for instance %s", info.ID), - Value: accessToken, - } + registerAccessToken, err := buildManagedRegisterAccessToken(info) + if err != nil { + return err + } + if registerAccessToken != nil { + registerReq.AccessToken = registerAccessToken } req := util.Request{Method: util.Verb_POST} @@ -100,6 +97,9 @@ func ConnectToManager() error { server, res, err := submitRequestToManager(&req) if err == nil && server != "" { if res.StatusCode == 200 || util.ContainStr(string(res.Body), "exists") { + if err := execPostRegisterHooks(server, res); err != nil { + return err + } log.Infof("success register to config manager: %v", string(server)) err := kv.AddValue(bucketName, []byte(global.Env().SystemConfig.NodeConfig.ID), []byte(util.GetLowPrecisionCurrentTime().String())) if err != nil { @@ -113,25 +113,57 @@ func ConnectToManager() error { return err } +func buildManagedRegisterAccessToken(info model.Instance) (*common.RegisterToken, error) { + if !common.SupportsManagedAccessToken(info.Application.Name) { + return nil, nil + } + accessToken, err := common.EnsureTokenInKeystore(common.AgentAccessTokenKeystoreKey) + if err != nil { + return nil, err + } + productName := strings.TrimSpace(info.Application.Name) + if productName == "" { + productName = "instance" + } + return &common.RegisterToken{ + Name: fmt.Sprintf("%s access token", info.ID), + Description: fmt.Sprintf("Console to %s access token for instance %s", productName, info.ID), + Value: accessToken, + }, nil +} + +func AddPostRegisterHook(hook func(server string, res *util.Result) error) { + if hook != nil { + postRegisterHooks = append(postRegisterHooks, hook) + } +} + +func execPostRegisterHooks(server string, res *util.Result) error { + for _, hook := range postRegisterHooks { + if err := hook(server, res); err != nil { + return err + } + } + return nil +} + func submitRequestToManager(req *util.Request) (string, *util.Result, error) { + return DoManagerRequest(req) +} + +func DoManagerRequest(req *util.Request) (string, *util.Result, error) { var err error var res *util.Result cfg := global.Env().SystemConfig.Configs - token, err := common.LoadTokenFromKeystore(common.ManagerTokenKeystoreKey) - if err != nil { + if err = applyManagerRequestAuth(req); err != nil { return "", nil, err } - if token != "" { - req.AddHeader("Authorization", "Bearer "+token) - } else if cfg.ManagerConfig.BasicAuth.Username != "" { - req.SetBasicAuth(cfg.ManagerConfig.BasicAuth.Username, cfg.ManagerConfig.BasicAuth.Password.Get()) - } for _, server := range cfg.Servers { req.Url, err = url.JoinPath(server, req.Path) if err != nil { continue } - res, err = util.ExecuteRequestWithCatchFlag(mTLSClient, req, true) + res, err = util.ExecuteRequestWithCatchFlag(getManagerHTTPClient(), req, true) if err != nil { continue } @@ -140,9 +172,45 @@ func submitRequestToManager(req *util.Request) (string, *util.Result, error) { return "", nil, err } +func applyManagerRequestAuth(req *util.Request) error { + cfg := global.Env().SystemConfig.Configs + if token := cfg.ManagerConfig.AccessToken.Get(); token != "" { + req.AddHeader(model.API_TOKEN, token) + return nil + } + token, err := common.LoadTokenFromKeystore(common.ManagerTokenKeystoreKey) + if err != nil { + return err + } + if token != "" { + req.AddHeader("Authorization", "Bearer "+token) + return nil + } + if cfg.ManagerConfig.BasicAuth.Username != "" { + req.SetBasicAuth(cfg.ManagerConfig.BasicAuth.Username, cfg.ManagerConfig.BasicAuth.Password.Get()) + } + return nil +} + var clientInitLock = sync.Once{} var mTLSClient *http.Client +func getManagerHTTPClient() *http.Client { + clientInitLock.Do(func() { + if global.Env().SystemConfig.Configs.Managed { + cfg := global.Env().GetHTTPClientConfig("configs", "") + if cfg != nil { + hClient, err := api.NewHTTPClient(cfg) + if err != nil { + panic(err) + } + mTLSClient = hClient + } + } + }) + return mTLSClient +} + func ListenConfigChanges() error { clientInitLock.Do(func() { @@ -180,7 +248,7 @@ func ListenConfigChanges() error { log.Debug("config sync request: ", string(util.MustToJSONBytes(req))) } - _, res, err := submitRequestToManager(&request) + _, res, err := DoManagerRequest(&request) if err != nil { log.Error("failed to submit request to config manager,", err) return diff --git a/modules/configs/client/client_test.go b/modules/configs/client/client_test.go new file mode 100644 index 000000000..1be5b4474 --- /dev/null +++ b/modules/configs/client/client_test.go @@ -0,0 +1,77 @@ +package client + +import ( + "strings" + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" + "infini.sh/framework/core/model" + "infini.sh/framework/core/util" + ucfg "infini.sh/framework/lib/go-ucfg" +) + +func TestApplyManagerRequestAuthUsesAccessTokenHeader(t *testing.T) { + oldConfigs := global.Env().SystemConfig.Configs + t.Cleanup(func() { + global.Env().SystemConfig.Configs = oldConfigs + }) + + global.Env().SystemConfig.Configs = config.ConfigsConfig{ + ManagerConfig: struct { + LocalConfigsRepoPath string `config:"local_configs_repo_path"` + BasicAuth config.BasicAuth `config:"basic_auth"` + AccessToken ucfg.SecretString `config:"access_token"` + }{ + AccessToken: ucfg.SecretString("manager-api-token"), + BasicAuth: config.BasicAuth{ + Username: "manager", + Password: ucfg.SecretString("secret"), + }, + }, + } + + req := &util.Request{} + if err := applyManagerRequestAuth(req); err != nil { + t.Fatalf("expected nil error, got %v", err) + } + + headers := req.AllHeaders() + if headers[model.API_TOKEN] != "manager-api-token" { + t.Fatalf("expected %s header to be set, got %#v", model.API_TOKEN, headers) + } + if auth := headers["Authorization"]; auth != "" { + t.Fatalf("expected no Authorization header, got %q", auth) + } +} + +func TestBuildManagedRegisterAccessToken(t *testing.T) { + t.Setenv("KEYSTORE_PATH", t.TempDir()) + + instance := model.Instance{} + instance.ID = "gateway-1" + instance.Application = env.Application{Name: "gateway"} + + registerToken, err := buildManagedRegisterAccessToken(instance) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if registerToken == nil || registerToken.Value == "" { + t.Fatalf("expected managed register token, got %#v", registerToken) + } + if !strings.Contains(registerToken.Description, "gateway") { + t.Fatalf("unexpected description: %q", registerToken.Description) + } + + other := model.Instance{} + other.ID = "other-1" + other.Application = env.Application{Name: "console"} + registerToken, err = buildManagedRegisterAccessToken(other) + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if registerToken != nil { + t.Fatalf("expected no managed register token, got %#v", registerToken) + } +} diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index 2c4a7f0ed..bb5780c09 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -27,7 +27,11 @@ package common -import "infini.sh/framework/core/model" +import ( + "strings" + + "infini.sh/framework/core/model" +) const REGISTER_API = "/instance/_register" const SYNC_API = "/configs/_sync" @@ -125,3 +129,12 @@ type InstanceSettings struct { ConfigFiles []string `config:"configs"` Secrets []string `config:"secrets"` } + +func SupportsManagedAccessToken(applicationName string) bool { + switch strings.ToLower(strings.TrimSpace(applicationName)) { + case "agent", "gateway": + return true + default: + return false + } +} diff --git a/modules/web/web.go b/modules/web/web.go index c46979881..4cada04a9 100755 --- a/modules/web/web.go +++ b/modules/web/web.go @@ -57,6 +57,9 @@ func (module *WebModule) Setup() { func (module *WebModule) Start() error { if global.Env().SystemConfig.WebAppConfig.Enabled { + if err := uis.ValidateServerExposureConfig(global.Env().SystemConfig); err != nil { + return err + } uis.StartWeb(global.Env().SystemConfig.WebAppConfig) } return nil From 6c906bbba7665a498d13d5777ad88fdd6cc4cf1a Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 10:18:11 +0800 Subject: [PATCH 071/137] improve: guide for init and setting for migration --- core/api/setting.go | 4 +++- core/api/setting_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 core/api/setting_test.go diff --git a/core/api/setting.go b/core/api/setting.go index 5456ee24c..6b224aaf1 100644 --- a/core/api/setting.go +++ b/core/api/setting.go @@ -29,6 +29,7 @@ package api import ( httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/global" "infini.sh/framework/core/util" "net/http" "sync" @@ -46,7 +47,8 @@ func init() { func appSettingsAPIHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { obj := util.MapStr{ - "auth_enabled": IsAuthEnable(), + "auth_enabled": IsAuthEnable(), + "setup_required": global.Env().SetupRequired(), } appSettings := GetAppSettings() obj.Merge(appSettings) diff --git a/core/api/setting_test.go b/core/api/setting_test.go new file mode 100644 index 000000000..23b416be6 --- /dev/null +++ b/core/api/setting_test.go @@ -0,0 +1,38 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" +) + +func TestAppSettingsAPIHandlerIncludesSetupRequired(t *testing.T) { + oldEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.PathConfig.Data = t.TempDir() + testEnv.EnableSetup(true) + global.RegisterEnv(testEnv) + defer global.RegisterEnv(oldEnv) + + req := httptest.NewRequest(http.MethodGet, "/setting/application", nil) + resp := httptest.NewRecorder() + + appSettingsAPIHandler(resp, req, nil) + + if resp.Code != http.StatusOK { + t.Fatalf("expected status %d, got %d", http.StatusOK, resp.Code) + } + + var body map[string]interface{} + if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + + if v, ok := body["setup_required"].(bool); !ok || !v { + t.Fatalf("expected setup_required=true, got %#v", body["setup_required"]) + } +} From c049643abe7e308eae37cb311c8cd55721a61892 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 13:10:29 +0800 Subject: [PATCH 072/137] fix: config file check and security public api --- app.go | 7 ++++++- core/util/fsutils.go | 34 +++++++++++++++++++++---------- core/util/fsutils_test.go | 38 +++++++++++++++++++++++++++++++++++ modules/security/rbac/init.go | 38 +++++++++++++++++++---------------- 4 files changed, 88 insertions(+), 29 deletions(-) diff --git a/app.go b/app.go index 67a18536a..896885ab0 100755 --- a/app.go +++ b/app.go @@ -215,7 +215,12 @@ func (app *App) initWithFlags() { app.configFile = app.environment.GetAppLowercaseName() + ".yml" } - app.configFile = util.TryGetFileAbsPath(app.configFile, app.environment.IgnoreOnConfigMissing) + resolvedConfigFile, err := util.GetFileAbsPath(app.configFile, app.environment.IgnoreOnConfigMissing) + if err != nil { + log.Errorf("failed to locate main config file [%v]: %v", app.configFile, err) + os.Exit(1) + } + app.configFile = resolvedConfigFile if !util.FileExists(app.configFile) { fmt.Println(errors.Errorf("main config file [%v] not exists", app.configFile)) diff --git a/core/util/fsutils.go b/core/util/fsutils.go index 3e20aa483..6e9733e55 100755 --- a/core/util/fsutils.go +++ b/core/util/fsutils.go @@ -302,11 +302,11 @@ func FileExtension(file string) string { return strings.ToLower(strings.TrimSpace(ext)) } -// Smart get file abs path. +// GetFileAbsPath resolves filePath to an absolute path when the file exists. // -// If all attempts fail, and `ignoreMissing` is set to `true`, this function -// returns `filePath` as-is. Otherwise, it panics. -func TryGetFileAbsPath(filePath string, ignoreMissing bool) string { +// If all attempts fail and ignoreMissing is true, it returns filePath as-is. +// Otherwise it returns an error describing the attempted paths. +func GetFileAbsPath(filePath string, ignoreMissing bool) (string, error) { // The paths that we tried attempts := []string{} @@ -317,7 +317,7 @@ func TryGetFileAbsPath(filePath string, ignoreMissing bool) string { */ if FileExists(filePath) { - return filePath + return filePath, nil } else { attempts = append(attempts, filePath) } @@ -327,7 +327,7 @@ func TryGetFileAbsPath(filePath string, ignoreMissing bool) string { */ absPathRelativeToWd, _ := filepath.Abs(filePath) if FileExists(absPathRelativeToWd) { - return absPathRelativeToWd + return absPathRelativeToWd, nil } else { attempts = append(attempts, absPathRelativeToWd) } @@ -341,7 +341,7 @@ func TryGetFileAbsPath(filePath string, ignoreMissing bool) string { absPathRelativeToExeDir := path.Join(exeDir, filePath) if FileExists(absPathRelativeToExeDir) { - return absPathRelativeToExeDir + return absPathRelativeToExeDir, nil } else { attempts = append(attempts, absPathRelativeToExeDir) } @@ -349,15 +349,27 @@ func TryGetFileAbsPath(filePath string, ignoreMissing bool) string { } /* - * All attempts failed. Panic if `ignoreMissing` is not set. Otherwise, - * return `filePath` as-is. + * All attempts failed. Return an error if `ignoreMissing` is not set. + * Otherwise, return `filePath` as-is. */ if !ignoreMissing { errorMsg := fmt.Sprintf("failed to absolutize path '%s', tried %v, but they do not exist", filePath, attempts) - panic(errors.New(errorMsg)) + return "", errors.New(errorMsg) } else { - return filePath + return filePath, nil + } +} + +// Smart get file abs path. +// +// If all attempts fail, and `ignoreMissing` is set to `true`, this function +// returns `filePath` as-is. Otherwise, it panics. +func TryGetFileAbsPath(filePath string, ignoreMissing bool) string { + absPath, err := GetFileAbsPath(filePath, ignoreMissing) + if err != nil { + panic(err) } + return absPath } func ListAllFiles(path string) ([]string, error) { diff --git a/core/util/fsutils_test.go b/core/util/fsutils_test.go index 5613590d8..1392e80ad 100755 --- a/core/util/fsutils_test.go +++ b/core/util/fsutils_test.go @@ -42,6 +42,7 @@ package util import ( "fmt" "github.com/stretchr/testify/assert" + "os" "path" "path/filepath" "testing" @@ -145,3 +146,40 @@ func TestNormalizeFolderPath(t *testing.T) { }) } } + +func TestGetFileAbsPathReturnsAbsolutePathForExistingFile(t *testing.T) { + tempDir := t.TempDir() + configPath := filepath.Join(tempDir, "console.yml") + err := os.WriteFile(configPath, []byte("name: console\n"), 0644) + assert.NoError(t, err) + + resolvedPath, err := GetFileAbsPath(configPath, false) + assert.NoError(t, err) + assert.Equal(t, configPath, resolvedPath) +} + +func TestGetFileAbsPathReturnsErrorForMissingFile(t *testing.T) { + missingPath := filepath.Join(t.TempDir(), "missing-console.yml") + + resolvedPath, err := GetFileAbsPath(missingPath, false) + assert.Error(t, err) + assert.Empty(t, resolvedPath) + assert.Contains(t, err.Error(), "failed to absolutize path") + assert.Contains(t, err.Error(), missingPath) +} + +func TestGetFileAbsPathReturnsOriginalPathWhenMissingIsIgnored(t *testing.T) { + missingPath := filepath.Join(t.TempDir(), "missing-console.yml") + + resolvedPath, err := GetFileAbsPath(missingPath, true) + assert.NoError(t, err) + assert.Equal(t, missingPath, resolvedPath) +} + +func TestTryGetFileAbsPathPanicsForMissingFile(t *testing.T) { + missingPath := filepath.Join(t.TempDir(), "missing-console.yml") + + assert.Panics(t, func() { + TryGetFileAbsPath(missingPath, false) + }) +} diff --git a/modules/security/rbac/init.go b/modules/security/rbac/init.go index 3b6290421..acdc7ce8a 100644 --- a/modules/security/rbac/init.go +++ b/modules/security/rbac/init.go @@ -19,23 +19,7 @@ func Init() { security.RegisterAuthenticationProvider(security.DefaultNativeAuthBackend, &provider) security.RegisterAuthorizationProvider(security.DefaultNativeAuthBackend, &provider) - api.HandleUIMethod(api.POST, "/account/replay_nonce", - api.RequireSecureTransport(IssueReplayNonce), - api.AllowPublicAccess(), - api.AllowOPTIONSS(), - api.Feature(api.FeatureCORS)) - - api.HandleUIMethod(api.POST, "/account/login/challenge", - api.RequireSecureTransport(LoginChallenge), - api.AllowPublicAccess(), - api.AllowOPTIONSS(), - api.Feature(api.FeatureCORS)) - - api.HandleUIMethod(api.POST, "/account/login", - api.RequireSecureTransport(Login), - api.AllowPublicAccess(), - api.AllowOPTIONSS(), - api.Feature(api.FeatureCORS)) + RegisterPublicUIAuthRoutes() orm.MustRegisterSchemaWithIndexName(&security.UserAccount{}, "app-users") orm.MustRegisterSchemaWithIndexName(&security.UserRole{}, "app-roles") @@ -84,3 +68,23 @@ func Init() { } } + +func RegisterPublicUIAuthRoutes() { + api.HandleUIMethod(api.POST, "/account/replay_nonce", + api.RequireSecureTransport(IssueReplayNonce), + api.AllowPublicAccess(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS)) + + api.HandleUIMethod(api.POST, "/account/login/challenge", + api.RequireSecureTransport(LoginChallenge), + api.AllowPublicAccess(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS)) + + api.HandleUIMethod(api.POST, "/account/login", + api.RequireSecureTransport(Login), + api.AllowPublicAccess(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS)) +} From e4dacd98dcce0ab6749a982329eab424a95fae57 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 13:12:01 +0800 Subject: [PATCH 073/137] fix: config file check and security public api --- app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.go b/app.go index 896885ab0..d89a08fa4 100755 --- a/app.go +++ b/app.go @@ -231,7 +231,7 @@ func (app *App) initWithFlags() { app.environment.SetConfigFile(app.configFile) - err := app.environment.InitPaths(app.configFile) + err = app.environment.InitPaths(app.configFile) if err != nil { panic(err) } From 2e16b2b3616fe3f61e56694d658ba125fad92e86 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 13:23:33 +0800 Subject: [PATCH 074/137] fix: ui method after api call with bridge --- core/api/protected_routes.go | 40 +++++++++++++++++++++++- core/api/web_test.go | 60 ++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/core/api/protected_routes.go b/core/api/protected_routes.go index 7353469cf..b10562c12 100644 --- a/core/api/protected_routes.go +++ b/core/api/protected_routes.go @@ -1,6 +1,10 @@ package api -import httprouter "infini.sh/framework/core/api/router" +import ( + "sort" + + httprouter "infini.sh/framework/core/api/router" +) type ProtectedAPIRoute struct { Method Method @@ -46,3 +50,37 @@ func RegisterProtectedRouterRoutes(router *httprouter.Router, routes []Protected router.Handle(string(route.Method), route.Path, handle) } } + +// RegisterMissingAPIMethodUIRoutes mirrors registered API method routes onto the +// web router only when no UI route already owns the same method/path. +func RegisterMissingAPIMethodUIRoutes(handle httprouter.Handle, options ...Option) { + if handle == nil { + return + } + + l.Lock() + routes := make([]ProtectedAPIRoute, 0) + for method, handlers := range registeredAPIMethodHandler { + for path := range handlers { + if shouldSkipEmbeddedAPIRoute(method, path) { + continue + } + routes = append(routes, ProtectedAPIRoute{ + Method: Method(method), + Path: path, + }) + } + } + l.Unlock() + + sort.Slice(routes, func(i, j int) bool { + if routes[i].Method == routes[j].Method { + return routes[i].Path < routes[j].Path + } + return routes[i].Method < routes[j].Method + }) + + for _, route := range routes { + HandleUIMethod(route.Method, route.Path, handle, options...) + } +} diff --git a/core/api/web_test.go b/core/api/web_test.go index 0e0d8bf0e..a0134738c 100644 --- a/core/api/web_test.go +++ b/core/api/web_test.go @@ -2,6 +2,7 @@ package api import ( "net/http" + "net/http/httptest" "testing" httprouter "infini.sh/framework/core/api/router" @@ -85,3 +86,62 @@ func TestShouldSkipEmbeddedAPIRoute(t *testing.T) { t.Fatal("expected unrelated API route not to be skipped") } } + +func TestRegisterMissingAPIMethodUIRoutesSkipsExistingUIRoutes(t *testing.T) { + originalAPIHandlers := registeredAPIMethodHandler + originalUIHandlers := registeredUIMethodHandler + originalServer := srv + t.Cleanup(func() { + registeredAPIMethodHandler = originalAPIHandlers + registeredUIMethodHandler = originalUIHandlers + srv = originalServer + }) + + registeredAPIMethodHandler = map[string]map[string]func(http.ResponseWriter, *http.Request, httprouter.Params){ + http.MethodGet: { + "/api-only": func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusAccepted) + }, + "/stats": func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusNoContent) + }, + }, + } + registeredUIMethodHandler = map[Method]map[string]RegisteredAPIHandler{ + GET: { + "/stats": { + Handler: func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusCreated) + }, + Options: &HandlerOptions{}, + }, + }, + } + + RegisterMissingAPIMethodUIRoutes(func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusOK) + }) + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = "127.0.0.1:0" + StartWeb(webCfg) + defer StopWeb(webCfg) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api-only", nil) + if err := ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve api-only ui route: %v", err) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected missing API route to be mirrored onto web, got %d", resp.Code) + } + + resp = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/stats", nil) + if err := ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve stats ui route: %v", err) + } + if resp.Code != http.StatusCreated { + t.Fatalf("expected existing UI route to win over mirrored API route, got %d", resp.Code) + } +} From fe252eb34331449e3945bf53fa089354877b6fb8 Mon Sep 17 00:00:00 2001 From: Medcl Date: Thu, 28 May 2026 10:58:38 +0800 Subject: [PATCH 075/137] chore: fix incorrect provider (#371) --- core/security/run_as.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/security/run_as.go b/core/security/run_as.go index 87ebb368e..5f3e2d0ae 100644 --- a/core/security/run_as.go +++ b/core/security/run_as.go @@ -8,12 +8,11 @@ import ( "context" ) -func RunAs(ctx context.Context, userID string) context.Context { +func RunAs(ctx context.Context,provider, userID string) context.Context { claims := UserSessionInfo{} claims.SetUserID(userID) - //claims.System = accessToken.System - claims.Provider = "run_as" + claims.Provider = provider claims.Login = userID claims.UserAssignedPermission = GetUserPermissions(&claims) From 624f0b982133d99a2f9e2c0164af18d0eef3d819 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 13:54:14 +0800 Subject: [PATCH 076/137] fix: add missing api method ui route --- core/api/protected_routes.go | 75 ++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 12 deletions(-) diff --git a/core/api/protected_routes.go b/core/api/protected_routes.go index b10562c12..01224ed2b 100644 --- a/core/api/protected_routes.go +++ b/core/api/protected_routes.go @@ -4,6 +4,7 @@ import ( "sort" httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/util" ) type ProtectedAPIRoute struct { @@ -51,36 +52,86 @@ func RegisterProtectedRouterRoutes(router *httprouter.Router, routes []Protected } } -// RegisterMissingAPIMethodUIRoutes mirrors registered API method routes onto the -// web router only when no UI route already owns the same method/path. -func RegisterMissingAPIMethodUIRoutes(handle httprouter.Handle, options ...Option) { - if handle == nil { +type MissingAPIMethodUIRoute struct { + Route ProtectedAPIRoute + Options *HandlerOptions +} + +func WalkMissingAPIMethodUIRoutes(walk func(route MissingAPIMethodUIRoute)) { + if walk == nil { return } l.Lock() - routes := make([]ProtectedAPIRoute, 0) + routes := make([]MissingAPIMethodUIRoute, 0) for method, handlers := range registeredAPIMethodHandler { for path := range handlers { if shouldSkipEmbeddedAPIRoute(method, path) { continue } - routes = append(routes, ProtectedAPIRoute{ - Method: Method(method), - Path: path, + var options *HandlerOptions + if registeredOptions, ok := apiOptions.Get(Method(method), path); ok { + options = cloneHandlerOptions(registeredOptions) + } + routes = append(routes, MissingAPIMethodUIRoute{ + Route: ProtectedAPIRoute{ + Method: Method(method), + Path: path, + }, + Options: options, }) } } l.Unlock() sort.Slice(routes, func(i, j int) bool { - if routes[i].Method == routes[j].Method { - return routes[i].Path < routes[j].Path + if routes[i].Route.Method == routes[j].Route.Method { + return routes[i].Route.Path < routes[j].Route.Path } - return routes[i].Method < routes[j].Method + return routes[i].Route.Method < routes[j].Route.Method }) for _, route := range routes { - HandleUIMethod(route.Method, route.Path, handle, options...) + walk(route) } } + +// RegisterMissingAPIMethodUIRoutes mirrors registered API method routes onto the +// web router only when no UI route already owns the same method/path. +func RegisterMissingAPIMethodUIRoutes(handle httprouter.Handle, options ...Option) { + if handle == nil { + return + } + + WalkMissingAPIMethodUIRoutes(func(route MissingAPIMethodUIRoute) { + HandleUIMethod(route.Route.Method, route.Route.Path, handle, options...) + }) +} + +func cloneHandlerOptions(options *HandlerOptions) *HandlerOptions { + if options == nil { + return nil + } + + cloned := *options + if options.RequirePermission != nil { + cloned.RequirePermission = append([]PermissionKey(nil), options.RequirePermission...) + } + if options.Tags != nil { + cloned.Tags = append([]string(nil), options.Tags...) + } + if options.Features != nil { + cloned.Features = map[string]bool{} + for key, value := range options.Features { + cloned.Features[key] = value + } + } + if options.Labels != nil { + cloned.Labels = util.MapStr{} + for key, value := range options.Labels { + cloned.Labels[key] = value + } + } + + return &cloned +} From 50e191a9cf8732f0464020130396be8c95cfcd0a Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 14:05:43 +0800 Subject: [PATCH 077/137] fix: add missing api method ui route for web --- core/api/web.go | 14 ++++++++++++++ core/api/web_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/core/api/web.go b/core/api/web.go index fd6674cbe..d785ecefb 100755 --- a/core/api/web.go +++ b/core/api/web.go @@ -486,9 +486,11 @@ func HandleUIMethod(method Method, pattern string, handler func(w http.ResponseW apiOptions.Register(method, pattern, opts) } + var hadPrevious bool if !opts.Override { //check previous handler previous, ok := registeredUIMethodHandler[method][pattern] + hadPrevious = ok if ok { if previous.Options.Priority > opts.Priority { log.Tracef("skip api: [%v] [%v], priority: [%v] < [%v]", method, pattern, opts.Priority, previous.Options.Priority) @@ -505,16 +507,28 @@ func HandleUIMethod(method Method, pattern string, handler func(w http.ResponseW myHandler := RegisteredAPIHandler{Handler: handler, Options: opts} registeredUIMethodHandler[method][pattern] = myHandler + registerLiveUIMethodHandler(method, pattern, myHandler, hadPrevious) if opts.AllowOPTIONS { m := registeredUIMethodHandler[OPTIONS] + hadOptionsPrevious := false if m == nil { registeredUIMethodHandler[OPTIONS] = map[string]RegisteredAPIHandler{} + } else { + _, hadOptionsPrevious = m[pattern] } registeredUIMethodHandler[OPTIONS][pattern] = myHandler + registerLiveUIMethodHandler(OPTIONS, pattern, myHandler, hadOptionsPrevious) } } +func registerLiveUIMethodHandler(method Method, pattern string, handler RegisteredAPIHandler, alreadyRegistered bool) { + if uiRouter == nil || alreadyRegistered { + return + } + uiRouter.Handle(string(method), pattern, getWrappedHandler(string(method), pattern, handler)) +} + // HandleWebSocketCommand register websocket command handler func HandleWebSocketCommand(command string, usage string, handler func(c *websocket.WebsocketConnection, array []string)) { diff --git a/core/api/web_test.go b/core/api/web_test.go index a0134738c..eef6a232f 100644 --- a/core/api/web_test.go +++ b/core/api/web_test.go @@ -91,10 +91,14 @@ func TestRegisterMissingAPIMethodUIRoutesSkipsExistingUIRoutes(t *testing.T) { originalAPIHandlers := registeredAPIMethodHandler originalUIHandlers := registeredUIMethodHandler originalServer := srv + originalRouter := uiRouter + originalServeMux := uiServeMux t.Cleanup(func() { registeredAPIMethodHandler = originalAPIHandlers registeredUIMethodHandler = originalUIHandlers srv = originalServer + uiRouter = originalRouter + uiServeMux = originalServeMux }) registeredAPIMethodHandler = map[string]map[string]func(http.ResponseWriter, *http.Request, httprouter.Params){ @@ -145,3 +149,36 @@ func TestRegisterMissingAPIMethodUIRoutesSkipsExistingUIRoutes(t *testing.T) { t.Fatalf("expected existing UI route to win over mirrored API route, got %d", resp.Code) } } + +func TestHandleUIMethodRegistersRouteAfterStartWeb(t *testing.T) { + originalUIHandlers := registeredUIMethodHandler + originalServer := srv + originalRouter := uiRouter + originalServeMux := uiServeMux + t.Cleanup(func() { + registeredUIMethodHandler = originalUIHandlers + srv = originalServer + uiRouter = originalRouter + uiServeMux = originalServeMux + }) + + registeredUIMethodHandler = map[Method]map[string]RegisteredAPIHandler{} + + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = "127.0.0.1:0" + StartWeb(webCfg) + defer StopWeb(webCfg) + + HandleUIMethod(GET, "/late-ui-route", func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { + w.WriteHeader(http.StatusAccepted) + }) + + resp := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/late-ui-route", nil) + if err := ServeRegisteredUIRequest(resp, req); err != nil { + t.Fatalf("serve late ui route: %v", err) + } + if resp.Code != http.StatusAccepted { + t.Fatalf("expected late ui route to be available after web start, got %d", resp.Code) + } +} From 46ed91976972b5486bf532c35ec8713b7b27f638 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 14:24:11 +0800 Subject: [PATCH 078/137] fix: install script with token sync --- core/api/api.go | 9 ++++++++- core/api/web.go | 3 +-- core/api/web_test.go | 17 +++++++++++++++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/core/api/api.go b/core/api/api.go index 52167412b..307b07143 100755 --- a/core/api/api.go +++ b/core/api/api.go @@ -120,7 +120,7 @@ func initializeAPI() { } // HandleAPIMethod register api handler -func HandleAPIMethod(method Method, pattern string, handler func(w http.ResponseWriter, req *http.Request, ps httprouter.Params)) { +func HandleAPIMethod(method Method, pattern string, handler func(w http.ResponseWriter, req *http.Request, ps httprouter.Params), options ...Option) { l.Lock() if registeredAPIMethodHandler == nil { registeredAPIMethodHandler = map[string]map[string]func(w http.ResponseWriter, req *http.Request, ps httprouter.Params){} @@ -132,6 +132,13 @@ func HandleAPIMethod(method Method, pattern string, handler func(w http.Response registeredAPIMethodHandler[m] = map[string]func(w http.ResponseWriter, req *http.Request, ps httprouter.Params){} } registeredAPIMethodHandler[m][pattern] = handler + if len(options) > 0 { + opts := &HandlerOptions{} + for _, option := range options { + option(opts) + } + apiOptions.Register(method, pattern, opts) + } l.Unlock() } diff --git a/core/api/web.go b/core/api/web.go index d785ecefb..e2e309a89 100755 --- a/core/api/web.go +++ b/core/api/web.go @@ -486,11 +486,10 @@ func HandleUIMethod(method Method, pattern string, handler func(w http.ResponseW apiOptions.Register(method, pattern, opts) } - var hadPrevious bool + _, hadPrevious := registeredUIMethodHandler[method][pattern] if !opts.Override { //check previous handler previous, ok := registeredUIMethodHandler[method][pattern] - hadPrevious = ok if ok { if previous.Options.Priority > opts.Priority { log.Tracef("skip api: [%v] [%v], priority: [%v] < [%v]", method, pattern, opts.Priority, previous.Options.Priority) diff --git a/core/api/web_test.go b/core/api/web_test.go index eef6a232f..68de6cb2a 100644 --- a/core/api/web_test.go +++ b/core/api/web_test.go @@ -1,6 +1,7 @@ package api import ( + "net" "net/http" "net/http/httptest" "testing" @@ -9,6 +10,18 @@ import ( "infini.sh/framework/core/config" ) +func newTestBinding(t *testing.T) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen on random port: %v", err) + } + defer listener.Close() + + return listener.Addr().String() +} + func TestWebsocketRegistrationPath(t *testing.T) { cfg := config.WebAppConfig{} cfg.WebsocketConfig.Enabled = true @@ -127,7 +140,7 @@ func TestRegisterMissingAPIMethodUIRoutesSkipsExistingUIRoutes(t *testing.T) { }) webCfg := config.WebAppConfig{} - webCfg.NetworkConfig.Binding = "127.0.0.1:0" + webCfg.NetworkConfig.Binding = newTestBinding(t) StartWeb(webCfg) defer StopWeb(webCfg) @@ -165,7 +178,7 @@ func TestHandleUIMethodRegistersRouteAfterStartWeb(t *testing.T) { registeredUIMethodHandler = map[Method]map[string]RegisteredAPIHandler{} webCfg := config.WebAppConfig{} - webCfg.NetworkConfig.Binding = "127.0.0.1:0" + webCfg.NetworkConfig.Binding = newTestBinding(t) StartWeb(webCfg) defer StopWeb(webCfg) From 7f611039df369d39867fa7d2a9d3eace87ed744c Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 15:09:58 +0800 Subject: [PATCH 079/137] fix: auth for instance stats --- modules/elastic/module.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/elastic/module.go b/modules/elastic/module.go index cf20510be..fa283aac5 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -128,6 +128,9 @@ func loadESBasedElasticConfig() []elastic.ElasticsearchConfig { return configs } query := elastic.SearchRequest{From: 0, Size: 1000} //TODO handle clusters beyond 1000 + query.Set("query", util.MapStr{ + "match_all": util.MapStr{}, + }) esClient := elastic.GetClient(systemID) result, err := esClient.Search(orm.GetIndexName(elastic.ElasticsearchConfig{}), &query) if err != nil { From 30b30b6a2b084967b701e14dfb08b7fc43ae58d8 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 15:30:52 +0800 Subject: [PATCH 080/137] fix: gateway access with username and password --- modules/configs/client/client.go | 40 ++++++------ modules/configs/client/client_test.go | 66 ++++++++++++++++++++ modules/elastic/metadata.go | 37 ++++++++++- modules/elastic/metadata_discovery_test.go | 38 ++++++++++++ modules/elastic/orm.go | 71 ++++++++++++++-------- 5 files changed, 205 insertions(+), 47 deletions(-) create mode 100644 modules/elastic/metadata_discovery_test.go diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index 009adb595..8e0ce3d52 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -192,38 +192,36 @@ func applyManagerRequestAuth(req *util.Request) error { return nil } -var clientInitLock = sync.Once{} +var managerHTTPClientInitLock = sync.Once{} +var configSyncInitLock = sync.Once{} var mTLSClient *http.Client -func getManagerHTTPClient() *http.Client { - clientInitLock.Do(func() { - if global.Env().SystemConfig.Configs.Managed { - cfg := global.Env().GetHTTPClientConfig("configs", "") - if cfg != nil { - hClient, err := api.NewHTTPClient(cfg) - if err != nil { - panic(err) - } - mTLSClient = hClient +func initManagerHTTPClient() { + managerHTTPClientInitLock.Do(func() { + if !global.Env().SystemConfig.Configs.Managed { + return + } + cfg := global.Env().GetHTTPClientConfig("configs", "") + if cfg != nil { + hClient, err := api.NewHTTPClient(cfg) + if err != nil { + panic(err) } + mTLSClient = hClient } }) +} + +func getManagerHTTPClient() *http.Client { + initManagerHTTPClient() return mTLSClient } func ListenConfigChanges() error { - - clientInitLock.Do(func() { + configSyncInitLock.Do(func() { if global.Env().SystemConfig.Configs.Managed { - cfg := global.Env().GetHTTPClientConfig("configs", "") - if cfg != nil { - hClient, err := api.NewHTTPClient(cfg) - if err != nil { - panic(err) - } - mTLSClient = hClient - } + initManagerHTTPClient() //init config sync listening req := common.ConfigSyncRequest{} diff --git a/modules/configs/client/client_test.go b/modules/configs/client/client_test.go index 1be5b4474..af46963f5 100644 --- a/modules/configs/client/client_test.go +++ b/modules/configs/client/client_test.go @@ -1,7 +1,13 @@ package client import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" "strings" + "sync" + "sync/atomic" "testing" "infini.sh/framework/core/config" @@ -75,3 +81,63 @@ func TestBuildManagedRegisterAccessToken(t *testing.T) { t.Fatalf("expected no managed register token, got %#v", registerToken) } } + +func TestListenConfigChangesStillSyncsAfterHTTPClientInit(t *testing.T) { + var syncRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/configs/_sync" { + t.Fatalf("unexpected request path: %s", r.URL.Path) + } + if r.Method != http.MethodPost { + t.Fatalf("unexpected request method: %s", r.Method) + } + syncRequests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"changed":false}`)) + })) + defer server.Close() + + tempDir := t.TempDir() + configDir := filepath.Join(tempDir, "configs") + if err := os.MkdirAll(configDir, 0o755); err != nil { + t.Fatalf("create config dir: %v", err) + } + mainConfigFile := filepath.Join(tempDir, "agent.yml") + if err := os.WriteFile(mainConfigFile, []byte("node:\n id: test-agent\n"), 0o644); err != nil { + t.Fatalf("write main config: %v", err) + } + + oldEnv := global.Env() + oldHTTPClientInitLock := managerHTTPClientInitLock + oldConfigSyncInitLock := configSyncInitLock + oldClient := mTLSClient + t.Cleanup(func() { + global.RegisterEnv(oldEnv) + managerHTTPClientInitLock = oldHTTPClientInitLock + configSyncInitLock = oldConfigSyncInitLock + mTLSClient = oldClient + }) + + testEnv := env.EmptyEnv() + testEnv.SystemConfig.Configs.Managed = true + testEnv.SystemConfig.Configs.Servers = []string{server.URL} + testEnv.SystemConfig.Configs.Interval = "30s" + testEnv.SystemConfig.PathConfig.Config = configDir + testEnv.SystemConfig.NodeConfig.ID = "test-agent" + testEnv.SetConfigFile(mainConfigFile) + global.RegisterEnv(testEnv) + + managerHTTPClientInitLock = sync.Once{} + configSyncInitLock = sync.Once{} + mTLSClient = nil + + if getManagerHTTPClient() == nil { + t.Fatal("expected manager HTTP client to initialize") + } + if err := ListenConfigChanges(); err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if syncRequests.Load() != 1 { + t.Fatalf("expected one immediate sync request, got %d", syncRequests.Load()) + } +} diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index 0bcb6bdf6..0e616eb67 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -50,6 +50,36 @@ import ( const elasticMetadataKVRetention = 30 * 24 * time.Hour +func shouldRegisterDiscoveredHostForAvailability(meta *elastic.ElasticsearchMetadata, host string) bool { + host = util.UnifyLocalAddress(strings.TrimSpace(host)) + if host == "" { + return false + } + + if meta == nil { + return true + } + if meta.Config == nil { + return true + } + if meta.Config.Host == "" && len(meta.Config.Hosts) == 0 && meta.Config.Endpoint == "" && len(meta.Config.Endpoints) == 0 { + return true + } + + seedHosts := meta.GetSeedHosts() + if len(seedHosts) == 0 { + return true + } + + for _, seedHost := range seedHosts { + if util.UnifyLocalAddress(strings.TrimSpace(seedHost)) == host { + return true + } + } + + return false +} + func (module *ElasticModule) clusterHealthCheck(clusterID string, force bool) { log.Tracef("execute health check for: %v", clusterID) @@ -58,6 +88,7 @@ func (module *ElasticModule) clusterHealthCheck(clusterID string, force bool) { if cfg == nil || !cfg.Enabled { return } + if !force && !cfg.Monitored { log.Tracef("skip health check for unmonitored cluster: %v", clusterID) return @@ -838,7 +869,11 @@ func (module *ElasticModule) updateNodeInfo(meta *elastic.ElasticsearchMetadata, //register host to do availability monitoring if discovery { for _, v := range *nodes { - elastic.GetOrInitHost(v.GetHttpPublishHost(), meta.Config.ID) + host := v.GetHttpPublishHost() + if !shouldRegisterDiscoveredHostForAvailability(meta, host) { + continue + } + elastic.GetOrInitHost(host, meta.Config.ID) } } diff --git a/modules/elastic/metadata_discovery_test.go b/modules/elastic/metadata_discovery_test.go new file mode 100644 index 000000000..4db2b96b6 --- /dev/null +++ b/modules/elastic/metadata_discovery_test.go @@ -0,0 +1,38 @@ +package elastic + +import ( + "testing" + + coreelastic "infini.sh/framework/core/elastic" + "infini.sh/framework/core/orm" +) + +func TestShouldRegisterDiscoveredHostForAvailabilityPrefersSeedHosts(t *testing.T) { + meta := &coreelastic.ElasticsearchMetadata{ + Config: &coreelastic.ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: "cluster-1"}, + Host: "192.168.3.8:9200", + Hosts: []string{"192.168.3.8:9200"}, + }, + } + + if shouldRegisterDiscoveredHostForAvailability(meta, "172.22.0.2:9200") { + t.Fatal("expected non-seed discovered host to be excluded from availability monitoring") + } + + if !shouldRegisterDiscoveredHostForAvailability(meta, "192.168.3.8:9200") { + t.Fatal("expected seed host to remain eligible for availability monitoring") + } +} + +func TestShouldRegisterDiscoveredHostForAvailabilityAllowsDiscoveryWithoutSeeds(t *testing.T) { + meta := &coreelastic.ElasticsearchMetadata{ + Config: &coreelastic.ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: "cluster-2"}, + }, + } + + if !shouldRegisterDiscoveredHostForAvailability(meta, "172.22.0.2:9200") { + t.Fatal("expected discovered host to be eligible when no seed hosts are configured") + } +} diff --git a/modules/elastic/orm.go b/modules/elastic/orm.go index 7da6b0bf6..25127e6de 100755 --- a/modules/elastic/orm.go +++ b/modules/elastic/orm.go @@ -46,6 +46,17 @@ type ElasticORM struct { Config common.ORMConfig } +func shouldRetrySearchWithoutCollapse(searchResponse *elastic.SearchResponse, collapseField string) bool { + if strings.TrimSpace(collapseField) == "" || searchResponse == nil || searchResponse.RawResult == nil { + return false + } + if searchResponse.RawResult.StatusCode != http.StatusBadRequest { + return false + } + body := string(searchResponse.RawResult.Body) + return strings.Contains(body, collapseField) && strings.Contains(body, "in order to collapse on") +} + var templateInited bool func InitTemplate(force bool) { @@ -552,6 +563,10 @@ func (handler *ElasticORM) Search(t interface{}, q *api.Query) (error, api.Resul } searchResponse, err = handler.Client.Search(indexName, &request) + if err == nil && shouldRetrySearchWithoutCollapse(searchResponse, q.CollapseField) { + request.Collapse = nil + searchResponse, err = handler.Client.Search(indexName, &request) + } } if err != nil { @@ -621,36 +636,38 @@ func (handler *ElasticORM) SearchWithResultItemMapper(resultArray interface{}, i searchResponse, err = handler.Client.SearchByTemplate(indexName, q.TemplatedQuery.TemplateID, q.TemplatedQuery.Parameters) } else { - request.Query = &elastic.Query{} - boolQuery := elastic.BoolQuery{} - - if q.Conds != nil && len(q.Conds) > 0 { - for _, cond := range q.Conds { - query := getQuery(cond) - switch cond.BoolType { - case api.Filter: - boolQuery.Filter = append(boolQuery.Filter, query) - case api.Must: - boolQuery.Must = append(boolQuery.Must, query) - case api.MustNot: - boolQuery.MustNot = append(boolQuery.MustNot, query) - case api.Should: - boolQuery.Should = append(boolQuery.Should, query) + if q.Filter != nil || q.Conds != nil && len(q.Conds) > 0 { + request.Query = &elastic.Query{} + boolQuery := elastic.BoolQuery{} + + if q.Conds != nil && len(q.Conds) > 0 { + for _, cond := range q.Conds { + query := getQuery(cond) + switch cond.BoolType { + case api.Filter: + boolQuery.Filter = append(boolQuery.Filter, query) + case api.Must: + boolQuery.Must = append(boolQuery.Must, query) + case api.MustNot: + boolQuery.MustNot = append(boolQuery.MustNot, query) + case api.Should: + boolQuery.Should = append(boolQuery.Should, query) + } } } - } - if q.Filter != nil { - filter := getQuery(q.Filter) - //temp fix for must_not filters - if q.Filter.BoolType == api.MustNot { - boolQuery.MustNot = append(boolQuery.MustNot, filter) - } else { - boolQuery.Filter = append(boolQuery.Filter, filter) + if q.Filter != nil { + filter := getQuery(q.Filter) + //temp fix for must_not filters + if q.Filter.BoolType == api.MustNot { + boolQuery.MustNot = append(boolQuery.MustNot, filter) + } else { + boolQuery.Filter = append(boolQuery.Filter, filter) + } } - } - request.Query.BoolQuery = &boolQuery + request.Query.BoolQuery = &boolQuery + } // Add sorting if specified if q.Sort != nil && len(*q.Sort) > 0 { @@ -661,6 +678,10 @@ func (handler *ElasticORM) SearchWithResultItemMapper(resultArray interface{}, i // Perform the search searchResponse, err = handler.Client.Search(indexName, &request) + if err == nil && shouldRetrySearchWithoutCollapse(searchResponse, q.CollapseField) { + request.Collapse = nil + searchResponse, err = handler.Client.Search(indexName, &request) + } } // Handle search errors From 8f9e46477d9788aa5e769f4e28b2b2c600fda966 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 15:53:52 +0800 Subject: [PATCH 081/137] fix: instance stats stuck the http request --- core/api/api.go | 23 +++++++++++++---- core/api/api_test.go | 40 +++++++++++++++++++++++++++++ core/api/web.go | 11 ++++++-- core/api/web_test.go | 61 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 7 deletions(-) diff --git a/core/api/api.go b/core/api/api.go index 307b07143..a7b3cc9fe 100755 --- a/core/api/api.go +++ b/core/api/api.go @@ -149,20 +149,33 @@ func ServeRegisteredAPIRequest(w http.ResponseWriter, req *http.Request) { localRouter.NotFound = notfoundHandler l.Lock() - defer l.Unlock() - + funcHandlers := make(map[string]func(http.ResponseWriter, *http.Request), len(registeredAPIFuncHandler)) for pattern, handler := range registeredAPIFuncHandler { + funcHandlers[pattern] = handler + } + methodHandlers := make(map[string]map[string]func(w http.ResponseWriter, req *http.Request, ps httprouter.Params), len(registeredAPIMethodHandler)) + for method, handlers := range registeredAPIMethodHandler { + cloned := make(map[string]func(w http.ResponseWriter, req *http.Request, ps httprouter.Params), len(handlers)) + for pattern, handler := range handlers { + cloned[pattern] = handler + } + methodHandlers[method] = cloned + } + filterSnapshot := append([]filter.Filter(nil), filters...) + l.Unlock() + + for pattern, handler := range funcHandlers { wrapped := handler - for _, f := range filters { + for _, f := range filterSnapshot { wrapped = f.FilterHttpHandlerFunc(pattern, wrapped) } localMux.HandleFunc(pattern, wrapped) } - for method, handlers := range registeredAPIMethodHandler { + for method, handlers := range methodHandlers { for pattern, handler := range handlers { wrapped := handler - for _, f := range filters { + for _, f := range filterSnapshot { wrapped = f.FilterHttpRouter(pattern, wrapped) } localRouter.Handle(method, pattern, wrapped) diff --git a/core/api/api_test.go b/core/api/api_test.go index d0ef493f0..81982743b 100644 --- a/core/api/api_test.go +++ b/core/api/api_test.go @@ -31,6 +31,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" httprouter "infini.sh/framework/core/api/router" ) @@ -134,3 +135,42 @@ func TestServeRegisteredAPIRequest(t *testing.T) { t.Fatalf("unexpected body: %s", recorder.Body.String()) } } + +func TestServeRegisteredAPIRequestAllowsNestedDispatch(t *testing.T) { + innerPath := fmt.Sprintf("/__copilot_test__/api/%s/inner", t.Name()) + outerPath := fmt.Sprintf("/__copilot_test__/api/%s/outer", t.Name()) + + HandleAPIMethod(GET, innerPath, func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write([]byte("inner-ok")) + }) + HandleAPIMethod(GET, outerPath, func(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { + innerReq := httptest.NewRequest(http.MethodGet, innerPath, nil) + innerRecorder := httptest.NewRecorder() + ServeRegisteredAPIRequest(innerRecorder, innerReq) + w.WriteHeader(innerRecorder.Code) + _, _ = w.Write(innerRecorder.Body.Bytes()) + }) + + req := httptest.NewRequest(http.MethodGet, outerPath, nil) + recorder := httptest.NewRecorder() + + done := make(chan struct{}) + go func() { + defer close(done) + ServeRegisteredAPIRequest(recorder, req) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("nested dispatch timed out") + } + + if recorder.Code != http.StatusAccepted { + t.Fatalf("unexpected status: %d", recorder.Code) + } + if recorder.Body.String() != "inner-ok" { + t.Fatalf("unexpected body: %s", recorder.Body.String()) + } +} diff --git a/core/api/web.go b/core/api/web.go index e2e309a89..334e6a887 100755 --- a/core/api/web.go +++ b/core/api/web.go @@ -30,6 +30,7 @@ package api import ( ctx "context" "crypto/tls" + "errors" "fmt" "net/http" _ "net/http/pprof" @@ -67,14 +68,19 @@ func ServeRegisteredUIRequest(w http.ResponseWriter, req *http.Request) error { func StopWeb(cfg config.WebAppConfig) { if srv != nil { - ctx1, cancel := ctx.WithTimeout(ctx.Background(), 10*time.Second) + ctx1, cancel := ctx.WithTimeout(ctx.Background(), webShutdownTimeout) defer cancel() err := srv.Shutdown(ctx1) if err != nil { - panic(err) + log.Warnf("graceful web shutdown timed out or failed: %v, forcing close", err) + closeErr := srv.Close() + if closeErr != nil && !errors.Is(closeErr, http.ErrServerClosed) { + log.Errorf("force closing web server failed: %v", closeErr) + } } log.Debug("stopping web server") + srv = nil } } @@ -406,6 +412,7 @@ func AddGlobalInterceptors(interceptors ...Interceptor) { } var srv *http.Server +var webShutdownTimeout = 10 * time.Second // RegisteredUIHandler is a hub for registered ui handler var registeredUIHandler map[string]http.Handler diff --git a/core/api/web_test.go b/core/api/web_test.go index 68de6cb2a..399cd5fb3 100644 --- a/core/api/web_test.go +++ b/core/api/web_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" httprouter "infini.sh/framework/core/api/router" "infini.sh/framework/core/config" @@ -195,3 +196,63 @@ func TestHandleUIMethodRegistersRouteAfterStartWeb(t *testing.T) { t.Fatalf("expected late ui route to be available after web start, got %d", resp.Code) } } + +func TestStopWebFallsBackToCloseWhenGracefulShutdownTimesOut(t *testing.T) { + originalUIHandlers := registeredUIMethodHandler + originalServer := srv + originalRouter := uiRouter + originalServeMux := uiServeMux + originalTimeout := webShutdownTimeout + t.Cleanup(func() { + registeredUIMethodHandler = originalUIHandlers + srv = originalServer + uiRouter = originalRouter + uiServeMux = originalServeMux + webShutdownTimeout = originalTimeout + }) + + registeredUIMethodHandler = map[Method]map[string]RegisteredAPIHandler{} + webCfg := config.WebAppConfig{} + webCfg.NetworkConfig.Binding = newTestBinding(t) + StartWeb(webCfg) + + started := make(chan struct{}) + release := make(chan struct{}) + HandleUIMethod(GET, "/shutdown-timeout", func(w http.ResponseWriter, req *http.Request, _ httprouter.Params) { + close(started) + select { + case <-release: + case <-req.Context().Done(): + } + w.WriteHeader(http.StatusOK) + }) + + clientDone := make(chan struct{}) + go func() { + defer close(clientDone) + _, _ = http.Get("http://" + webCfg.NetworkConfig.Binding + "/shutdown-timeout") + }() + + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("request did not reach blocking handler") + } + + webShutdownTimeout = 50 * time.Millisecond + + stopDone := make(chan struct{}) + go func() { + defer close(stopDone) + StopWeb(webCfg) + }() + + select { + case <-stopDone: + case <-time.After(2 * time.Second): + t.Fatal("StopWeb did not return after graceful shutdown timeout") + } + + close(release) + <-clientDone +} From 5d971167689ad7fb2f506a96f07763f7e726b100 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 17:24:04 +0800 Subject: [PATCH 082/137] fix: migration with token auth by default --- modules/security/rbac/account_login.go | 4 +- modules/security/rbac/account_login_test.go | 50 +++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go index 2e630fb97..a5f3bbdd0 100644 --- a/modules/security/rbac/account_login.go +++ b/modules/security/rbac/account_login.go @@ -99,7 +99,7 @@ func LoginChallenge(w http.ResponseWriter, r *http.Request, ps httprouter.Params return } - exists, user, err := GetUserByLogin(login) + exists, user, err := security.GetUserByLogin(login) if err != nil { api.WriteError(w, err.Error(), http.StatusInternalServerError) return @@ -124,7 +124,7 @@ func Login(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { } usedChallenge := req.ChallengeID != "" || req.Proof != "" - exists, user, err := GetUserByLogin(login) + exists, user, err := security.GetUserByLogin(login) if err != nil { api.WriteError(w, err.Error(), http.StatusInternalServerError) return diff --git a/modules/security/rbac/account_login_test.go b/modules/security/rbac/account_login_test.go index 6b4b60a75..09976ebcd 100644 --- a/modules/security/rbac/account_login_test.go +++ b/modules/security/rbac/account_login_test.go @@ -24,6 +24,8 @@ package rbac import ( + "bytes" + "encoding/json" "errors" "net/http" "net/http/httptest" @@ -36,6 +38,28 @@ import ( type testAccountPasswordLoginProvider struct{} +type testChallengeAuthenticationBackend struct{} + +func (testChallengeAuthenticationBackend) GetUserByID(id string) (bool, *security.UserAccount, error) { + return false, nil, nil +} + +func (testChallengeAuthenticationBackend) GetUserByLogin(login string) (bool, *security.UserAccount, error) { + if login != "bridge-admin" { + return false, nil, nil + } + user := &security.UserAccount{Email: "bridge-admin"} + if err := security.SetPassword(user, "StrongPassw0rd!"); err != nil { + return false, nil, err + } + user.ID = "bridge-admin-id" + return true, user, nil +} + +func (testChallengeAuthenticationBackend) CreateUser(name, login, password string, force bool) (*security.UserAccount, error) { + return nil, nil +} + func (testAccountPasswordLoginProvider) AuthenticateByPassword(login, password string) (*security.UserSessionInfo, error) { if login != "ldap-user" || password != "StrongPassw0rd!" { return nil, nil @@ -187,6 +211,32 @@ func TestBuildLoginChallengeResponseReturnsChallenge(t *testing.T) { } } +func TestLoginChallengeUsesRegisteredAuthenticationBackend(t *testing.T) { + security.RegisterAuthenticationProvider("test-login-challenge-provider", testChallengeAuthenticationBackend{}) + + body := bytes.NewBufferString(`{"login":"bridge-admin"}`) + req := httptest.NewRequest(http.MethodPost, "/account/login/challenge", body) + req.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + + LoginChallenge(recorder, req, nil) + + if recorder.Code != http.StatusOK { + t.Fatalf("expected 200 response, got %d: %s", recorder.Code, recorder.Body.String()) + } + + var resp map[string]interface{} + if err := json.Unmarshal(recorder.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if got := resp["method"]; got != security.PasswordChallengeMethod { + t.Fatalf("expected challenge method from registered provider, got %v", got) + } + if resp["challenge_id"] == "" { + t.Fatal("expected challenge id from registered provider") + } +} + // Legacy password clients keep working even before they learn the replay-nonce preflight. func TestValidateReplayNonceAllowsLegacyPasswordLoginWithoutNonce(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/account/login", nil) From 5f2b5117d892b4e5c1cd13bc5a54749dfa378c09 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 29 May 2026 21:34:19 +0800 Subject: [PATCH 083/137] fix: disk sync with queue --- modules/queue/disk_queue/diskqueue.go | 13 ++-- modules/queue/disk_queue/diskqueue_test.go | 85 ++++++++++++++++++++++ 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/modules/queue/disk_queue/diskqueue.go b/modules/queue/disk_queue/diskqueue.go index e3f8c9278..0842e1939 100644 --- a/modules/queue/disk_queue/diskqueue.go +++ b/modules/queue/disk_queue/diskqueue.go @@ -273,11 +273,7 @@ func (d *DiskBasedQueue) getWriteTimeout(payloadSize int) time.Duration { // Close cleans up the queue and persists metadata func (d *DiskBasedQueue) Close() error { - err := d.exit(false) - if err != nil { - return err - } - return d.sync() + return d.exit(false) } // Destroy cleans up all data for the specified queue @@ -337,6 +333,11 @@ func (d *DiskBasedQueue) exit(deleted bool) error { // ensure that ioLoop has exited <-d.exitSyncChan + var syncErr error + if !deleted { + syncErr = d.sync() + } + close(d.depthChan) if d.readFile != nil { @@ -349,7 +350,7 @@ func (d *DiskBasedQueue) exit(deleted bool) error { d.writeFile = nil } - return nil + return syncErr } // Empty destructively clears out any pending data in the queue diff --git a/modules/queue/disk_queue/diskqueue_test.go b/modules/queue/disk_queue/diskqueue_test.go index b4c522da5..b75f3e7e7 100644 --- a/modules/queue/disk_queue/diskqueue_test.go +++ b/modules/queue/disk_queue/diskqueue_test.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "os" "path/filepath" + "sync" "testing" "time" @@ -46,6 +47,90 @@ func TestGetWriteTimeoutCapsAtMaximum(t *testing.T) { } } +func TestClosePersistsUnsyncedWritesBeforeClosingFiles(t *testing.T) { + env1 := EmptyEnv() + env1.SystemConfig.PathConfig.Data = t.TempDir() + global.RegisterEnv(env1) + + queueName := "close-persists-unsynced" + cfg := &DiskQueueConfig{ + MinMsgSize: 1, + MaxMsgSize: 1024, + MaxBytesPerFile: 1024 * 1024, + SyncEveryRecords: 1 << 20, + SyncTimeoutInMS: 1 << 20, + ReadChanBuffer: 1, + WriteChanBuffer: 1, + } + normalizeDiskQueueConfig(cfg) + + dataPath := GetDataPath(queueName) + if err := os.MkdirAll(dataPath, 0o755); err != nil { + t.Fatalf("failed to create queue data dir: %v", err) + } + + dq := &DiskBasedQueue{ + name: queueName, + dataPath: dataPath, + cfg: cfg, + readChan: make(chan []byte, cfg.ReadChanBuffer), + depthChan: make(chan int64), + writeChan: make(chan []byte, cfg.WriteChanBuffer), + writeResponseChan: make(chan WriteResponse), + emptyChan: make(chan int), + emptyResponseChan: make(chan error), + exitChan: make(chan int), + exitSyncChan: make(chan int, 1), + consumersInReading: sync.Map{}, + } + go dq.ioLoop() + + res := dq.Put([]byte("hello")) + if res.Error != nil { + t.Fatalf("failed to put queue message: %v", res.Error) + } + if dq.writeFile == nil { + t.Fatalf("expected queue write file to remain open before close") + } + + if err := dq.Close(); err != nil { + t.Fatalf("failed to close queue: %v", err) + } + + reopened := &DiskBasedQueue{ + name: queueName, + dataPath: dataPath, + cfg: cfg, + readChan: make(chan []byte, cfg.ReadChanBuffer), + depthChan: make(chan int64), + writeChan: make(chan []byte, cfg.WriteChanBuffer), + writeResponseChan: make(chan WriteResponse), + emptyChan: make(chan int), + emptyResponseChan: make(chan error), + exitChan: make(chan int), + exitSyncChan: make(chan int, 1), + consumersInReading: sync.Map{}, + } + if err := reopened.retrieveMetaData(); err != nil { + t.Fatalf("failed to reload queue metadata: %v", err) + } + t.Cleanup(func() { + _ = os.RemoveAll(dataPath) + }) + + if depth := reopened.Depth(); depth != 1 { + t.Fatalf("expected reopened queue depth 1, got %d", depth) + } + + message, err := reopened.readOne() + if err != nil { + t.Fatalf("failed to read reopened queue message: %v", err) + } + if string(message) != "hello" { + t.Fatalf("expected reopened queue payload %q, got %q", "hello", string(message)) + } +} + func TestResetOffsetSkipsMissingSegmentsUpToCurrentWriteSegment(t *testing.T) { env1 := EmptyEnv() env1.SystemConfig.PathConfig.Data = t.TempDir() From b16b2dda08687a2fbb4731f778913228abe09138 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 30 May 2026 06:38:43 +0800 Subject: [PATCH 084/137] fix: console restart with bad file --- modules/queue/disk_queue/diskqueue.go | 155 +++++++++++++++++++++ modules/queue/disk_queue/diskqueue_test.go | 81 ++++++++++- 2 files changed, 234 insertions(+), 2 deletions(-) diff --git a/modules/queue/disk_queue/diskqueue.go b/modules/queue/disk_queue/diskqueue.go index 0842e1939..37e05c0a4 100644 --- a/modules/queue/disk_queue/diskqueue.go +++ b/modules/queue/disk_queue/diskqueue.go @@ -143,6 +143,9 @@ func NewDiskQueueByConfig(name, dataPath string, cfg *DiskQueueConfig) *DiskBase if err != nil && !os.IsNotExist(err) { log.Errorf("diskqueue(%s) failed to retrieveMetaData - %s", d.name, err) } + if repairErr := d.repairTailMetadata(); repairErr != nil { + log.Errorf("diskqueue(%s) failed to repair tail metadata - %s", d.name, repairErr) + } _, ok := queue.GetConsumerConfigsByQueueID(d.name) if ok { @@ -713,6 +716,158 @@ func (d *DiskBasedQueue) retrieveMetaData() error { return nil } +type segmentScanResult struct { + validEnd int64 + totalMessages int64 + messagesBeforeReadPos int64 + messagesBeforeWritePos int64 + readBoundary int64 +} + +func scanSegmentFileTail(file *os.File, cfg *DiskQueueConfig, readPos, writePos int64) (segmentScanResult, error) { + result := segmentScanResult{} + if file == nil || cfg == nil { + return result, nil + } + + if _, err := file.Seek(0, 0); err != nil { + return result, err + } + + reader := bufio.NewReader(file) + var offset int64 + + for { + var msgSize int32 + if err := binary.Read(reader, binary.BigEndian, &msgSize); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return result, nil + } + return result, err + } + + if msgSize < cfg.MinMsgSize || msgSize > cfg.MaxMsgSize { + return result, nil + } + + payloadSize := int64(msgSize) + if _, err := io.CopyN(io.Discard, reader, payloadSize); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + return result, nil + } + return result, err + } + + offset += 4 + payloadSize + result.validEnd = offset + result.totalMessages++ + if offset <= readPos { + result.messagesBeforeReadPos++ + result.readBoundary = offset + } + if offset <= writePos { + result.messagesBeforeWritePos++ + } + } +} + +func (d *DiskBasedQueue) repairTailMetadata() error { + if d == nil || d.cfg == nil { + return nil + } + if d.writeSegmentNum == 0 && d.writePos == 0 { + return nil + } + + fileName := d.GetFileName(d.writeSegmentNum) + if !util.FileExists(fileName) { + return nil + } + + file, err := os.OpenFile(fileName, os.O_RDWR, 0600) + if err != nil { + return err + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return err + } + + readPos := int64(0) + if d.readSegmentFileNum == d.writeSegmentNum { + readPos = d.readPos + } + + scan, err := scanSegmentFileTail(file, d.cfg, readPos, d.writePos) + if err != nil { + return err + } + + newWritePos := scan.validEnd + newReadPos := readPos + if d.readSegmentFileNum == d.writeSegmentNum { + if newReadPos > newWritePos { + newReadPos = newWritePos + } + if scan.readBoundary < newReadPos { + newReadPos = scan.readBoundary + } + } + + oldUnreadInTail := scan.messagesBeforeWritePos + newUnreadInTail := scan.totalMessages + if d.readSegmentFileNum == d.writeSegmentNum { + oldUnreadInTail -= scan.messagesBeforeReadPos + newUnreadInTail -= scan.messagesBeforeReadPos + } + + changed := false + if stat.Size() != scan.validEnd { + if err := file.Truncate(scan.validEnd); err != nil { + return err + } + if err := file.Sync(); err != nil { + return err + } + log.Warnf("diskqueue(%s) truncated tail segment %s from %d to %d bytes during startup recovery", + d.name, fileName, stat.Size(), scan.validEnd) + changed = true + } + + if d.writePos != newWritePos { + d.writePos = newWritePos + changed = true + } + + if d.readSegmentFileNum == d.writeSegmentNum && d.readPos != newReadPos { + d.readPos = newReadPos + d.nextReadPos = newReadPos + changed = true + } + + if delta := newUnreadInTail - oldUnreadInTail; delta != 0 { + d.depth += delta + if d.depth < 0 { + d.depth = 0 + } + changed = true + } + + if d.nextReadFileNum == d.writeSegmentNum && d.nextReadPos > d.writePos { + d.nextReadPos = d.writePos + changed = true + } + + if !changed { + return nil + } + + d.needSync = true + return d.persistMetaData() +} + // persistMetaData atomically writes state to the filesystem func (d *DiskBasedQueue) persistMetaData() error { d.metaLock.Lock() diff --git a/modules/queue/disk_queue/diskqueue_test.go b/modules/queue/disk_queue/diskqueue_test.go index b75f3e7e7..b38d59ef3 100644 --- a/modules/queue/disk_queue/diskqueue_test.go +++ b/modules/queue/disk_queue/diskqueue_test.go @@ -59,7 +59,7 @@ func TestClosePersistsUnsyncedWritesBeforeClosingFiles(t *testing.T) { MaxBytesPerFile: 1024 * 1024, SyncEveryRecords: 1 << 20, SyncTimeoutInMS: 1 << 20, - ReadChanBuffer: 1, + ReadChanBuffer: 0, WriteChanBuffer: 1, } normalizeDiskQueueConfig(cfg) @@ -118,7 +118,7 @@ func TestClosePersistsUnsyncedWritesBeforeClosingFiles(t *testing.T) { _ = os.RemoveAll(dataPath) }) - if depth := reopened.Depth(); depth != 1 { + if depth := reopened.depth; depth != 1 { t.Fatalf("expected reopened queue depth 1, got %d", depth) } @@ -131,6 +131,83 @@ func TestClosePersistsUnsyncedWritesBeforeClosingFiles(t *testing.T) { } } +func TestRepairTailMetadataTruncatesIncompleteTailOnStartup(t *testing.T) { + env1 := EmptyEnv() + env1.SystemConfig.PathConfig.Data = t.TempDir() + global.RegisterEnv(env1) + + queueName := "repair-tail-startup" + cfg := &DiskQueueConfig{ + MinMsgSize: 1, + MaxMsgSize: 1024, + MaxBytesPerFile: 1024 * 1024, + } + normalizeDiskQueueConfig(cfg) + + fileName := GetFileName(queueName, 0) + if err := os.MkdirAll(filepath.Dir(fileName), 0o755); err != nil { + t.Fatalf("failed to create queue dir: %v", err) + } + + payload := []byte("hello") + file, err := os.Create(fileName) + if err != nil { + t.Fatalf("failed to create tail file: %v", err) + } + if err := binary.Write(file, binary.BigEndian, int32(len(payload))); err != nil { + t.Fatalf("failed to write payload size: %v", err) + } + if _, err := file.Write(payload); err != nil { + t.Fatalf("failed to write payload: %v", err) + } + if _, err := file.Write([]byte{0x7f, 0xff}); err != nil { + t.Fatalf("failed to append corrupt tail: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("failed to close tail file: %v", err) + } + + dq := &DiskBasedQueue{ + name: queueName, + dataPath: GetDataPath(queueName), + cfg: cfg, + readSegmentFileNum: 0, + writeSegmentNum: 0, + readPos: 0, + nextReadPos: 0, + writePos: int64(4 + len(payload) + 2), + depth: 1, + } + + if err := dq.repairTailMetadata(); err != nil { + t.Fatalf("failed to repair tail metadata: %v", err) + } + + expectedWritePos := int64(4 + len(payload)) + if dq.writePos != expectedWritePos { + t.Fatalf("expected write pos %d after repair, got %d", expectedWritePos, dq.writePos) + } + if dq.depth != 1 { + t.Fatalf("expected queue depth to remain 1 after repair, got %d", dq.depth) + } + + stat, err := os.Stat(fileName) + if err != nil { + t.Fatalf("failed to stat repaired tail file: %v", err) + } + if stat.Size() != expectedWritePos { + t.Fatalf("expected repaired tail file size %d, got %d", expectedWritePos, stat.Size()) + } + + message, err := dq.readOne() + if err != nil { + t.Fatalf("failed to read message after repair: %v", err) + } + if string(message) != "hello" { + t.Fatalf("expected repaired payload %q, got %q", "hello", string(message)) + } +} + func TestResetOffsetSkipsMissingSegmentsUpToCurrentWriteSegment(t *testing.T) { env1 := EmptyEnv() env1.SystemConfig.PathConfig.Data = t.TempDir() From a03595ae0ad886aa8477ada4ff8cc5810921bcc6 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 30 May 2026 06:45:49 +0800 Subject: [PATCH 085/137] fix: console restart with bad file by comsumer --- plugins/queue/consumer/consumer.go | 92 ++++++++++++++----------- plugins/queue/consumer/consumer_test.go | 37 ++++++++++ 2 files changed, 89 insertions(+), 40 deletions(-) create mode 100644 plugins/queue/consumer/consumer_test.go diff --git a/plugins/queue/consumer/consumer.go b/plugins/queue/consumer/consumer.go index 5d768d19c..0da0b479f 100755 --- a/plugins/queue/consumer/consumer.go +++ b/plugins/queue/consumer/consumer.go @@ -28,6 +28,7 @@ import ( "infini.sh/framework/core/errors" "infini.sh/framework/core/locker" "runtime" + "strings" "sync" "time" @@ -191,20 +192,45 @@ func (processor *QueueConsumerProcessor) Name() string { return name } +func getRecoveredMessage(r interface{}) string { + switch v := r.(type) { + case error: + return v.Error() + case runtime.Error: + return v.Error() + case string: + return v + default: + return fmt.Sprint(v) + } +} + +func isExpectedQueueShutdownPanic(message string, contexts ...*pipeline.Context) bool { + normalized := strings.ToLower(strings.TrimSpace(message)) + if normalized == "" || !strings.Contains(normalized, "module closed") { + return false + } + if global.ShuttingDown() { + return true + } + for _, ctx := range contexts { + if ctx != nil && (ctx.IsCanceled() || ctx.IsFailed()) { + return true + } + } + return false +} + func (processor *QueueConsumerProcessor) Process(c *pipeline.Context) error { defer func() { if !global.Env().IsDebug { if r := recover(); r != nil { - var v string - switch r.(type) { - case error: - v = r.(error).Error() - case runtime.Error: - v = r.(runtime.Error).Error() - case string: - v = r.(string) + v := getRecoveredMessage(r) + if isExpectedQueueShutdownPanic(v, c) { + log.Debug("queue consumer processor stopped during shutdown,", v) + } else { + log.Error("error in consumer processor,", v) } - log.Error("error in consumer processor,", v) } } log.Debug("exit consumer processor") @@ -221,16 +247,12 @@ func (processor *QueueConsumerProcessor) Process(c *pipeline.Context) error { defer func() { if !global.Env().IsDebug { if r := recover(); r != nil { - var v string - switch r.(type) { - case error: - v = r.(error).Error() - case runtime.Error: - v = r.(runtime.Error).Error() - case string: - v = r.(string) + v := getRecoveredMessage(r) + if isExpectedQueueShutdownPanic(v, c) { + log.Debug("queue processor stopped during shutdown,", v) + } else { + log.Error("error in queue processor,", v) } - log.Error("error in queue processor,", v) } } processor.detectorRunning = false @@ -412,16 +434,12 @@ func (processor *QueueConsumerProcessor) NewSlicedWorker(ctx *pipeline.Context, defer func() { if !global.Env().IsDebug { if r := recover(); r != nil { - var v string - switch r.(type) { - case error: - v = r.(error).Error() - case runtime.Error: - v = r.(runtime.Error).Error() - case string: - v = r.(string) + v := getRecoveredMessage(r) + if isExpectedQueueShutdownPanic(v, ctx, parentContext) { + log.Debugf("consumer processor stopped during shutdown, queue:%v, slice_id:%v, %v", qConfig.ID, sliceID, v) + } else { + log.Errorf("error in consumer processor, %v, queue:%v, slice_id:%v", v, qConfig.ID, sliceID) } - log.Errorf("error in consumer processor, %v, queue:%v, slice_id:%v", v, qConfig.ID, sliceID) } } processor.inFlightQueueConfigs.Delete(key) @@ -476,21 +494,15 @@ func (processor *QueueConsumerProcessor) NewSlicedWorker(ctx *pipeline.Context, defer log.Debugf("exit worker[%v], queue:[%v], slice_id:%v", workerID, qConfig.ID, sliceID) if !global.Env().IsDebug { if r := recover(); r != nil { - var v string - switch r.(type) { - case error: - v = r.(error).Error() - case runtime.Error: - v = r.(runtime.Error).Error() - case string: - v = r.(string) - } - if v != "empty queue" { + v := getRecoveredMessage(r) + if isExpectedQueueShutdownPanic(v, ctx, parentContext) { + log.Debugf("worker[%v], queue:[%v], slice:[%v] stopped during shutdown, offset:[%v]->[%v], %v", workerID, qConfig.ID, sliceID, initOffset, offset, v) + } else if v != "empty queue" { log.Errorf("worker[%v], queue:[%v], slice:[%v], offset:[%v]->[%v],%v", workerID, qConfig.ID, sliceID, initOffset, offset, v) ctx.Failed(fmt.Errorf("panic in slice worker: %+v", r)) - } - if parentContext != nil { - parentContext.RecordError(fmt.Errorf("panic in slice worker: %+v", r)) + if parentContext != nil { + parentContext.RecordError(fmt.Errorf("panic in slice worker: %+v", r)) + } } } } diff --git a/plugins/queue/consumer/consumer_test.go b/plugins/queue/consumer/consumer_test.go new file mode 100644 index 000000000..b6fbda166 --- /dev/null +++ b/plugins/queue/consumer/consumer_test.go @@ -0,0 +1,37 @@ +package consumer + +import ( + "context" + "testing" + + "infini.sh/framework/core/pipeline" +) + +func TestIsExpectedQueueShutdownPanicRequiresShutdownSignal(t *testing.T) { + ctx := &pipeline.Context{Context: context.Background()} + if isExpectedQueueShutdownPanic("module closed", ctx) { + t.Fatal("expected module closed without shutdown or cancellation to remain an error") + } + if isExpectedQueueShutdownPanic("boom", ctx) { + t.Fatal("expected unrelated panic message to remain an error") + } +} + +func TestIsExpectedQueueShutdownPanicTreatsCanceledContextAsExpected(t *testing.T) { + baseCtx, cancel := context.WithCancel(context.Background()) + ctx := &pipeline.Context{Context: baseCtx} + cancel() + + if !isExpectedQueueShutdownPanic("module closed", ctx) { + t.Fatal("expected module closed during context cancellation to be treated as shutdown noise") + } +} + +func TestGetRecoveredMessage(t *testing.T) { + if got := getRecoveredMessage("boom"); got != "boom" { + t.Fatalf("unexpected string recovery message: %q", got) + } + if got := getRecoveredMessage(context.Canceled); got != context.Canceled.Error() { + t.Fatalf("unexpected error recovery message: %q", got) + } +} From 2109e3314c2a237381e2af2ec02029c3289df693 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 30 May 2026 17:05:03 +0800 Subject: [PATCH 086/137] improve: add sync publish address --- core/api/api.go | 13 ++++++++++++- core/api/api_test.go | 21 +++++++++++++++++++++ core/api/web.go | 1 + 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/core/api/api.go b/core/api/api.go index a7b3cc9fe..65f1c1eff 100755 --- a/core/api/api.go +++ b/core/api/api.go @@ -34,6 +34,7 @@ import ( "net" "net/http" "runtime" + "strings" "sync" "time" @@ -194,9 +195,18 @@ var rootKey *rsa.PrivateKey var rootCertPEM []byte var apiConfig *config.APIConfig - var listenAddress string +func syncRuntimePublishAddress(networkConfig *config.NetworkConfig, actualAddr string) { + if networkConfig == nil || strings.TrimSpace(actualAddr) == "" { + return + } + if strings.TrimSpace(networkConfig.Publish) != "" { + return + } + networkConfig.Publish = actualAddr +} + var notfoundHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { rw.Write([]byte("{\"message\":\"not_found\"}")) rw.WriteHeader(404) @@ -268,6 +278,7 @@ func StartAPI() { if err != nil { panic(err) } + syncRuntimePublishAddress(&apiConfig.NetworkConfig, l.Addr().String()) router.NotFound = notfoundHandler diff --git a/core/api/api_test.go b/core/api/api_test.go index 81982743b..fd6807d8e 100644 --- a/core/api/api_test.go +++ b/core/api/api_test.go @@ -34,6 +34,7 @@ import ( "time" httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/config" ) func TestStripPrefix(t *testing.T) { @@ -174,3 +175,23 @@ func TestServeRegisteredAPIRequestAllowsNestedDispatch(t *testing.T) { t.Fatalf("unexpected body: %s", recorder.Body.String()) } } + +func TestSyncRuntimePublishAddressUsesActualListenAddressWhenUnset(t *testing.T) { + cfg := config.NetworkConfig{} + + syncRuntimePublishAddress(&cfg, "0.0.0.0:2901") + + if cfg.Publish != "0.0.0.0:2901" { + t.Fatalf("expected runtime publish address to be updated, got %q", cfg.Publish) + } +} + +func TestSyncRuntimePublishAddressPreservesExplicitPublishAddress(t *testing.T) { + cfg := config.NetworkConfig{Publish: "gateway.example:8443"} + + syncRuntimePublishAddress(&cfg, "0.0.0.0:2901") + + if cfg.Publish != "gateway.example:8443" { + t.Fatalf("expected explicit publish address to be preserved, got %q", cfg.Publish) + } +} diff --git a/core/api/web.go b/core/api/web.go index 334e6a887..ea72494fa 100755 --- a/core/api/web.go +++ b/core/api/web.go @@ -165,6 +165,7 @@ func StartWeb(cfg config.WebAppConfig) { } else { bindAddress = cfg.NetworkConfig.GetBindingAddr() } + syncRuntimePublishAddress(&cfg.NetworkConfig, bindAddress) handler := context.ClearHandler(uiRouter) if cfg.Gzip.Enabled { From 68ef9e0e83917d535dc43745181f99a983da594d Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 30 May 2026 17:28:11 +0800 Subject: [PATCH 087/137] fix: https with nginx proxy --- modules/security/account/refresh.go | 9 ++++++++- modules/security/rbac/init.go | 7 ++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/modules/security/account/refresh.go b/modules/security/account/refresh.go index ea000d9e9..14b0e92d4 100644 --- a/modules/security/account/refresh.go +++ b/modules/security/account/refresh.go @@ -15,7 +15,14 @@ import ( ) func init() { - api.HandleUIMethod(api.POST, "/account/refresh", api.RequireSecureTransport(Refresh), api.RequireLogin(), api.AllowOPTIONSS(), api.Feature(api.FeatureCORS)) + api.HandleUIMethod( + api.POST, + "/account/refresh", + api.RequireSecureTransport(Refresh, api.SecureTransportOptions{TrustForwardHeaders: true}), + api.RequireLogin(), + api.AllowOPTIONSS(), + api.Feature(api.FeatureCORS), + ) } // Refresh reissues an access token for the current session user while reloading the diff --git a/modules/security/rbac/init.go b/modules/security/rbac/init.go index acdc7ce8a..a5aed809f 100644 --- a/modules/security/rbac/init.go +++ b/modules/security/rbac/init.go @@ -70,20 +70,21 @@ func Init() { } func RegisterPublicUIAuthRoutes() { + secureViaProxy := api.SecureTransportOptions{TrustForwardHeaders: true} api.HandleUIMethod(api.POST, "/account/replay_nonce", - api.RequireSecureTransport(IssueReplayNonce), + api.RequireSecureTransport(IssueReplayNonce, secureViaProxy), api.AllowPublicAccess(), api.AllowOPTIONSS(), api.Feature(api.FeatureCORS)) api.HandleUIMethod(api.POST, "/account/login/challenge", - api.RequireSecureTransport(LoginChallenge), + api.RequireSecureTransport(LoginChallenge, secureViaProxy), api.AllowPublicAccess(), api.AllowOPTIONSS(), api.Feature(api.FeatureCORS)) api.HandleUIMethod(api.POST, "/account/login", - api.RequireSecureTransport(Login), + api.RequireSecureTransport(Login, secureViaProxy), api.AllowPublicAccess(), api.AllowOPTIONSS(), api.Feature(api.FeatureCORS)) From 0bf767a0a3a3507ec845252ce17da3fd14f7245a Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 30 May 2026 21:49:31 +0800 Subject: [PATCH 088/137] fix: agent register publish network address --- core/api/api.go | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/core/api/api.go b/core/api/api.go index 65f1c1eff..c13674508 100755 --- a/core/api/api.go +++ b/core/api/api.go @@ -196,6 +196,34 @@ var rootCertPEM []byte var apiConfig *config.APIConfig var listenAddress string +var resolveRuntimePublishIPv4 = util.GetIntranetIP + +func normalizeRuntimePublishAddress(actualAddr string) string { + actualAddr = strings.TrimSpace(actualAddr) + if actualAddr == "" { + return actualAddr + } + + host, port, err := net.SplitHostPort(actualAddr) + if err != nil { + return actualAddr + } + + normalizedHost := strings.Trim(strings.TrimSpace(host), "[]") + if normalizedHost != "" { + ip := net.ParseIP(normalizedHost) + if normalizedHost != util.AnyAddress && (ip == nil || !ip.IsUnspecified()) { + return actualAddr + } + } + + ipv4, err := resolveRuntimePublishIPv4() + if err != nil || strings.TrimSpace(ipv4) == "" { + return actualAddr + } + + return net.JoinHostPort(ipv4, port) +} func syncRuntimePublishAddress(networkConfig *config.NetworkConfig, actualAddr string) { if networkConfig == nil || strings.TrimSpace(actualAddr) == "" { @@ -204,7 +232,7 @@ func syncRuntimePublishAddress(networkConfig *config.NetworkConfig, actualAddr s if strings.TrimSpace(networkConfig.Publish) != "" { return } - networkConfig.Publish = actualAddr + networkConfig.Publish = normalizeRuntimePublishAddress(actualAddr) } var notfoundHandler = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { From 93e67ba806cb0b18400d4b41bcf9d0b7303c0bcf Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 30 May 2026 21:49:49 +0800 Subject: [PATCH 089/137] fix: agent register publish network address test --- core/api/api_test.go | 46 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/core/api/api_test.go b/core/api/api_test.go index fd6807d8e..70ae597a9 100644 --- a/core/api/api_test.go +++ b/core/api/api_test.go @@ -177,15 +177,41 @@ func TestServeRegisteredAPIRequestAllowsNestedDispatch(t *testing.T) { } func TestSyncRuntimePublishAddressUsesActualListenAddressWhenUnset(t *testing.T) { + oldResolver := resolveRuntimePublishIPv4 + resolveRuntimePublishIPv4 = func() (string, error) { + return "192.168.3.185", nil + } + t.Cleanup(func() { + resolveRuntimePublishIPv4 = oldResolver + }) + cfg := config.NetworkConfig{} syncRuntimePublishAddress(&cfg, "0.0.0.0:2901") - if cfg.Publish != "0.0.0.0:2901" { + if cfg.Publish != "192.168.3.185:2901" { t.Fatalf("expected runtime publish address to be updated, got %q", cfg.Publish) } } +func TestSyncRuntimePublishAddressNormalizesIPv6UnspecifiedHost(t *testing.T) { + oldResolver := resolveRuntimePublishIPv4 + resolveRuntimePublishIPv4 = func() (string, error) { + return "192.168.3.185", nil + } + t.Cleanup(func() { + resolveRuntimePublishIPv4 = oldResolver + }) + + cfg := config.NetworkConfig{} + + syncRuntimePublishAddress(&cfg, "[::]:2901") + + if cfg.Publish != "192.168.3.185:2901" { + t.Fatalf("expected ipv6 unspecified runtime publish address to use ipv4, got %q", cfg.Publish) + } +} + func TestSyncRuntimePublishAddressPreservesExplicitPublishAddress(t *testing.T) { cfg := config.NetworkConfig{Publish: "gateway.example:8443"} @@ -195,3 +221,21 @@ func TestSyncRuntimePublishAddressPreservesExplicitPublishAddress(t *testing.T) t.Fatalf("expected explicit publish address to be preserved, got %q", cfg.Publish) } } + +func TestSyncRuntimePublishAddressPreservesConcreteListenAddress(t *testing.T) { + oldResolver := resolveRuntimePublishIPv4 + resolveRuntimePublishIPv4 = func() (string, error) { + return "192.168.3.185", nil + } + t.Cleanup(func() { + resolveRuntimePublishIPv4 = oldResolver + }) + + cfg := config.NetworkConfig{} + + syncRuntimePublishAddress(&cfg, "10.0.0.8:2901") + + if cfg.Publish != "10.0.0.8:2901" { + t.Fatalf("expected concrete runtime publish address to be preserved, got %q", cfg.Publish) + } +} From 0510d95c75ed2d7d94fb61eb95a44e7316df1dbd Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 31 May 2026 07:11:33 +0800 Subject: [PATCH 090/137] improve: edpoint with schema and log debug reduce --- core/config/config.go | 4 +- .../elastic/bulk_indexing/bulk_indexing.go | 94 ++++++++++++++++--- plugins/queue/consumer/consumer.go | 12 ++- 3 files changed, 91 insertions(+), 19 deletions(-) diff --git a/core/config/config.go b/core/config/config.go index 2ddf8de3b..7d51b3ad1 100755 --- a/core/config/config.go +++ b/core/config/config.go @@ -270,7 +270,7 @@ func LoadEnvVariablesFromConfig(configObject *Config) (map[string]interface{}, e return nil, err } - log.Debugf("config contain variables, try to parse with environments") + log.Tracef("config contains variables, parsing with environments") environs := os.Environ() obj := map[string]interface{}{} @@ -345,7 +345,7 @@ func internalLoadFile(path string) (*Config, error) { } - log.Debugf("load config file '%v'", path) + log.Tracef("load config file '%v'", path) return pCfg, err } diff --git a/plugins/elastic/bulk_indexing/bulk_indexing.go b/plugins/elastic/bulk_indexing/bulk_indexing.go index 02544a5fe..936fcb9d3 100755 --- a/plugins/elastic/bulk_indexing/bulk_indexing.go +++ b/plugins/elastic/bulk_indexing/bulk_indexing.go @@ -78,6 +78,63 @@ type BulkIndexingProcessor struct { bulkBufferPool *elastic.BulkBufferPool } +const bulkLogSampleLimit = 5 + +func summarizeBulkLogValues(values []string) string { + if len(values) == 0 { + return "[]" + } + + limit := bulkLogSampleLimit + if len(values) < limit { + limit = len(values) + } + + sample := values[:limit] + if len(values) > limit { + return fmt.Sprintf("%v...(and %d more)", sample, len(values)-limit) + } + + return fmt.Sprintf("%v", sample) +} + +func summarizeBulkDetailItem(item elastic.BulkDetailItem) string { + parts := make([]string, 0, 2) + if len(item.Documents) > 0 { + parts = append(parts, fmt.Sprintf("documents=%d sample=%s", len(item.Documents), summarizeBulkLogValues(item.Documents))) + } + if len(item.Reasons) > 0 { + parts = append(parts, fmt.Sprintf("reasons=%d sample=%s", len(item.Reasons), summarizeBulkLogValues(item.Reasons))) + } + if len(parts) == 0 { + return "empty" + } + return strings.Join(parts, ", ") +} + +func summarizeBulkResult(bulkResult *elastic.BulkResult) string { + if bulkResult == nil { + return "" + } + + parts := []string{ + fmt.Sprintf( + "summary={success:%d invalid:%d failure:%d}", + bulkResult.Summary.Success.Count, + bulkResult.Summary.Invalid.Count, + bulkResult.Summary.Failure.Count, + ), + fmt.Sprintf("error=%v", bulkResult.Error), + fmt.Sprintf("error_msgs=%d sample=%s", len(bulkResult.ErrorMsgs), summarizeBulkLogValues(bulkResult.ErrorMsgs)), + fmt.Sprintf("codes=%d", len(bulkResult.Codes)), + fmt.Sprintf("indices=%d", len(bulkResult.Indices)), + fmt.Sprintf("actions=%d", len(bulkResult.Actions)), + fmt.Sprintf("detail={failure:%s, invalid:%s}", summarizeBulkDetailItem(bulkResult.Detail.Failure), summarizeBulkDetailItem(bulkResult.Detail.Invalid)), + } + + return strings.Join(parts, ", ") +} + var queueOwners sync.Map type Config struct { @@ -236,7 +293,11 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { } } if processor.bulkStats != nil { - log.Debugf( + logFn := log.Tracef + if processor.bulkStats.Summary.Invalid.Count > 0 || processor.bulkStats.Summary.Failure.Count > 0 || len(processor.bulkStats.ErrorMsgs) > 0 { + logFn = log.Debugf + } + logFn( "exit bulk indexing processor, success=%d, invalid=%d, failure=%d, error_msgs=%d", processor.bulkStats.Summary.Success.Count, processor.bulkStats.Summary.Invalid.Count, @@ -244,7 +305,7 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { len(processor.bulkStats.ErrorMsgs), ) } else { - log.Debug("exit bulk indexing processor") + log.Trace("exit bulk indexing processor") } }() @@ -276,7 +337,7 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { } } processor.detectorRunning = false - log.Debug("exit detector for active queue") + log.Trace("exit detector for active queue") processor.wg.Done() }() @@ -336,7 +397,11 @@ func (processor *BulkIndexingProcessor) Process(c *pipeline.Context) error { util.MapLength(&processor.inFlightQueueConfigs), ) { if processor.bulkStats != nil { - log.Debugf( + logFn := log.Tracef + if processor.bulkStats.Summary.Invalid.Count > 0 || processor.bulkStats.Summary.Failure.Count > 0 || len(processor.bulkStats.ErrorMsgs) > 0 { + logFn = log.Debugf + } + logFn( "active queue detector idle exit, success=%d, invalid=%d, failure=%d, inflight=%d", processor.bulkStats.Summary.Success.Count, processor.bulkStats.Summary.Invalid.Count, @@ -918,12 +983,10 @@ READ_DOCS: consumerConfig.KeepActive() messages, timeout, err := consumerInstance.FetchMessages(ctx1, consumerConfig.FetchMaxMessages) stats.IncrementBy("queue", qConfig.ID+".msg_fetched_from_queue", int64(len(messages))) - if err != nil || len(messages) > 0 { - if qConfig.Name == "bulk_requests" { - log.Tracef("slice worker, worker:[%v], [%v][%v][%v][%v] fetched message:%v,ctx:%v,timeout:%v,err:%v", workerID, qConfig.Name, consumerConfig.Group, consumerConfig.Name, sliceID, len(messages), ctx1.String(), timeout, err) - } else { - log.Debugf("slice worker, worker:[%v], [%v][%v][%v][%v] fetched message:%v,ctx:%v,timeout:%v,err:%v", workerID, qConfig.Name, consumerConfig.Group, consumerConfig.Name, sliceID, len(messages), ctx1.String(), timeout, err) - } + if err != nil { + log.Debugf("slice worker, worker:[%v], [%v][%v][%v][%v] fetched message:%v,ctx:%v,timeout:%v,err:%v", workerID, qConfig.Name, consumerConfig.Group, consumerConfig.Name, sliceID, len(messages), ctx1.String(), timeout, err) + } else if len(messages) > 0 { + log.Tracef("slice worker, worker:[%v], [%v][%v][%v][%v] fetched message:%v,ctx:%v,timeout:%v,err:%v", workerID, qConfig.Name, consumerConfig.Group, consumerConfig.Name, sliceID, len(messages), ctx1.String(), timeout, err) } if err != nil { if strings.Contains(err.Error(), "dirty_read") || err.Error() == "EOF" || err.Error() == "unexpected EOF" { @@ -1233,11 +1296,18 @@ func (processor *BulkIndexingProcessor) submitBulkRequest(ctx *pipeline.Context, if bulkResult != nil { msg = bulkResult.Detail } - log.Warnf("elasticsearch [%v], stats:%v, detail: %v, err:%v", meta.Config.Name, statsMap, msg, err) + log.Warnf( + "elasticsearch [%v], stats:%v, detail:{failure:%s, invalid:%s}, err:%v", + meta.Config.Name, + statsMap, + summarizeBulkDetailItem(msg.Failure), + summarizeBulkDetailItem(msg.Invalid), + err, + ) } if global.Env().IsDebug { - log.Debug(tag, ", ", meta.Config.Name, ", ", host, ", stats: ", statsMap, ", count: ", count, ", size: ", util.ByteSize(uint64(size)), ", elapsed: ", time.Since(start), ", continue: ", continueRequest, ", bulkResult: ", bulkResult) + log.Debug(tag, ", ", meta.Config.Name, ", ", host, ", stats: ", statsMap, ", count: ", count, ", size: ", util.ByteSize(uint64(size)), ", elapsed: ", time.Since(start), ", continue: ", continueRequest, ", bulkResult: ", summarizeBulkResult(bulkResult)) } else { if processor.config.VerboseBulkResult { log.Info("queue:", qConfig.Name, ", ", meta.Config.Name, ", ", host, ", stats: ", statsMap, ", count: ", count, ", size: ", util.ByteSize(uint64(size)), ", elapsed: ", time.Since(start), ", continue: ", continueRequest) diff --git a/plugins/queue/consumer/consumer.go b/plugins/queue/consumer/consumer.go index 0da0b479f..91ca5a9a9 100755 --- a/plugins/queue/consumer/consumer.go +++ b/plugins/queue/consumer/consumer.go @@ -380,7 +380,7 @@ func (processor *QueueConsumerProcessor) HandleQueueConfig(qConfig *queue.QueueC continue } else { var workerID = util.GetUUID() - log.Debugf("starting worker:[%v], queue:[%v], slice_id:%v", workerID, qConfig.Name, sliceID) + log.Tracef("starting worker:[%v], queue:[%v], slice_id:%v", workerID, qConfig.Name, sliceID) processor.wg.Add(1) contextForWorker := pipeline.Context{} @@ -491,7 +491,7 @@ func (processor *QueueConsumerProcessor) NewSlicedWorker(ctx *pipeline.Context, defer xxHashPool.Put(xxHash) defer func() { - defer log.Debugf("exit worker[%v], queue:[%v], slice_id:%v", workerID, qConfig.ID, sliceID) + defer log.Tracef("exit worker[%v], queue:[%v], slice_id:%v", workerID, qConfig.ID, sliceID) if !global.Env().IsDebug { if r := recover(); r != nil { v := getRecoveredMessage(r) @@ -597,8 +597,10 @@ READ_DOCS: } consumerConfig.KeepActive() messages, timeout, err := consumerInstance.FetchMessages(ctx1, consumerConfig.FetchMaxMessages) - if global.Env().IsDebug { + if err != nil { log.Debugf("[%v] slice_worker, [%v][%v] consume message:%v,ctx:%v,timeout:%v,err:%v", qConfig.Name, consumerConfig.Name, sliceID, len(messages), ctx1.String(), timeout, err) + } else if global.Env().IsDebug && len(messages) > 0 { + log.Tracef("[%v] slice_worker, [%v][%v] consume message:%v,ctx:%v,timeout:%v,err:%v", qConfig.Name, consumerConfig.Name, sliceID, len(messages), ctx1.String(), timeout, err) } if err != nil { @@ -719,12 +721,12 @@ CLEAN_BUFFER: if processor.config.QuitNeedTag && processor.config.QuitNeedTagName != "" && !ctx.HasTag(processor.config.QuitNeedTagName) { time.Sleep(1 * time.Second) - log.Debug("EOF without quit tag, sleep 1s: ", qConfig.Name) + log.Trace("EOF without quit tag, sleep 1s: ", qConfig.Name) goto READ_DOCS } ctx.CancelTask() - log.Debug("EOF, cancel task: ", qConfig.Name) + log.Trace("EOF, cancel task: ", qConfig.Name) return } From 8903c1e76a01e589da4ef30c6f1dbd5e8cc522de Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 31 May 2026 07:34:29 +0800 Subject: [PATCH 091/137] improve: log reduce with error user login --- modules/security/rbac/account_login.go | 16 ++++++++++++++++ modules/security/rbac/account_login_test.go | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go index a5f3bbdd0..be79791ee 100644 --- a/modules/security/rbac/account_login.go +++ b/modules/security/rbac/account_login.go @@ -47,6 +47,19 @@ var ( errMissingPassword = errors.New("password is required") ) +func shouldCollapseLoginError(err error) bool { + if err == nil { + return false + } + + switch strings.ToLower(strings.TrimSpace(err.Error())) { + case "user not found": + return true + default: + return false + } +} + // accountLoginRequest accepts both the framework-native "login" field and the aliases // already used by existing clients while challenge login is rolled out incrementally. type accountLoginRequest struct { @@ -227,6 +240,9 @@ func authenticateLogin(user *security.UserAccount, login, password, challengeID, sessionUser, err := security.AuthenticateAccountPasswordLogin(login, password) if err != nil { + if shouldCollapseLoginError(err) { + return false, nil, nil, errInvalidLoginCredentials + } return false, nil, nil, err } if sessionUser != nil { diff --git a/modules/security/rbac/account_login_test.go b/modules/security/rbac/account_login_test.go index 09976ebcd..c9efbf75b 100644 --- a/modules/security/rbac/account_login_test.go +++ b/modules/security/rbac/account_login_test.go @@ -37,6 +37,7 @@ import ( ) type testAccountPasswordLoginProvider struct{} +type testMissingUserAccountPasswordLoginProvider struct{} type testChallengeAuthenticationBackend struct{} @@ -74,6 +75,14 @@ func (testAccountPasswordLoginProvider) AuthenticateByPassword(login, password s return sessionUser, nil } +func (testMissingUserAccountPasswordLoginProvider) AuthenticateByPassword(login, password string) (*security.UserSessionInfo, error) { + if login != "missing-user" { + return nil, nil + } + + return nil, errors.New("user not found") +} + // The request payload accepts multiple historical login field names during rollout. func TestAccountLoginRequestNormalizedLogin(t *testing.T) { req := accountLoginRequest{ @@ -176,6 +185,15 @@ func TestAuthenticateLoginFallsBackToRegisteredPasswordProvider(t *testing.T) { } } +func TestAuthenticateLoginCollapsesMissingUserProviderError(t *testing.T) { + security.RegisterAccountPasswordLoginProvider("test-account-login-missing-user", testMissingUserAccountPasswordLoginProvider{}) + + _, _, _, err := authenticateLogin(nil, "missing-user", "StrongPassw0rd!", "", "") + if !errors.Is(err, errInvalidLoginCredentials) { + t.Fatalf("expected invalid credential error for missing user, got %v", err) + } +} + // Older accounts intentionally advertise plain login until their verifier is available. func TestBuildLoginChallengeResponseFallsBackToPlain(t *testing.T) { user := &security.UserAccount{Email: "admin@example.org"} From 57545fe0dd30b0cc8fd1b6bd0ad2addc14a601e0 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 31 May 2026 07:40:11 +0800 Subject: [PATCH 092/137] improve: monitor log reduce with error user login --- core/elastic/actions.go | 14 +++++++++++++- core/elastic/actions_test.go | 17 +++++++++++++++++ core/vfs/static.go | 2 +- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/core/elastic/actions.go b/core/elastic/actions.go index 5ff110ff2..3e562470b 100644 --- a/core/elastic/actions.go +++ b/core/elastic/actions.go @@ -140,13 +140,25 @@ func (meta *ElasticsearchMetadata) IsAvailable() bool { clusterID = meta.Config.Name } if rate.GetRateLimiter("cluster_available_check", clusterID, 1, 1, 30*time.Second).Allow() { - log.Debugf("elasticsearch [%v] is unavailable: clusterAvailable=false", meta.Config.Name) + if meta.shouldTraceUnavailableReason() { + log.Tracef("elasticsearch [%v] is unavailable: clusterAvailable=false", meta.Config.Name) + } else { + log.Debugf("elasticsearch [%v] is unavailable: clusterAvailable=false", meta.Config.Name) + } } return false } return true } +func (meta *ElasticsearchMetadata) shouldTraceUnavailableReason() bool { + if meta == nil || meta.Config == nil { + return false + } + + return !meta.Config.Monitored +} + func (meta *ElasticsearchMetadata) Init(health bool) { meta.clusterAvailable = health if health && meta.Health == nil { diff --git a/core/elastic/actions_test.go b/core/elastic/actions_test.go index f7d2ef830..d558608b2 100644 --- a/core/elastic/actions_test.go +++ b/core/elastic/actions_test.go @@ -88,3 +88,20 @@ func TestGetActiveHostFallsBackToCachedDiscoveredHostWhenSeedUnavailable(t *test t.Fatalf("expected discovered host %q when seed host is unavailable, got %q", discoveredHost, got) } } + +func TestShouldTraceUnavailableReasonForUnmonitoredCluster(t *testing.T) { + meta := &ElasticsearchMetadata{ + Config: &ElasticsearchConfig{ + Monitored: false, + }, + } + + if !meta.shouldTraceUnavailableReason() { + t.Fatal("expected unmonitored cluster to trace unavailable reason") + } + + meta.Config.Monitored = true + if meta.shouldTraceUnavailableReason() { + t.Fatal("expected monitored cluster to keep debug unavailable reason") + } +} diff --git a/core/vfs/static.go b/core/vfs/static.go index 170b02d66..0b8e75cb7 100755 --- a/core/vfs/static.go +++ b/core/vfs/static.go @@ -103,7 +103,7 @@ func (fs StaticFS) Open(name string) (http.File, error) { } } - log.Debug("local file not found,", localFile) + log.Trace("local file not found,", localFile) } if fs.SkipVFS { From 32120e11c5eb2289a8c73c4713639cb5d3cc42c2 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 31 May 2026 08:50:41 +0800 Subject: [PATCH 093/137] improve: default close access log --- cmd/vfs/main.go | 2 +- core/config/system.go | 1 + modules/security/http_filters/logging.go | 8 +++ modules/security/http_filters/logging_test.go | 67 +++++++++++++++++++ 4 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 modules/security/http_filters/logging_test.go diff --git a/cmd/vfs/main.go b/cmd/vfs/main.go index 3a27f2ba0..745ea98dd 100755 --- a/cmd/vfs/main.go +++ b/cmd/vfs/main.go @@ -304,7 +304,7 @@ func (vfs StaticFS) Open(name string) (http.File, error) { } } - log.Debug("local file not found,", localFile) + log.Trace("local file not found,", localFile) } if vfs.SkipVFS{ diff --git a/core/config/system.go b/core/config/system.go index 63fd640d3..e6f59dba0 100755 --- a/core/config/system.go +++ b/core/config/system.go @@ -361,6 +361,7 @@ type WebAppConfig struct { //same with API Config Enabled bool `config:"enabled"` + AccessLog bool `config:"access_log_enabled"` TLSConfig TLSConfig `config:"tls"` NetworkConfig NetworkConfig `config:"network"` CrossDomain struct { diff --git a/modules/security/http_filters/logging.go b/modules/security/http_filters/logging.go index 50bfbf3db..0c5d9efc7 100644 --- a/modules/security/http_filters/logging.go +++ b/modules/security/http_filters/logging.go @@ -56,6 +56,10 @@ func getAccessLogHandler() *rotate.RotateWriter { return accessLogHandler } +func isAccessLogEnabled() bool { + return global.Env().SystemConfig != nil && global.Env().SystemConfig.WebAppConfig.AccessLog +} + func (f *LoggingFilter) GetPriority() int { // Lower priority values execute first (higher precedence) return 0 @@ -68,6 +72,10 @@ func (f *LoggingFilter) ApplyFilter( next httprouter.Handle, ) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + if !isAccessLogEnabled() { + next(w, r, ps) + return + } start := time.Now() diff --git a/modules/security/http_filters/logging_test.go b/modules/security/http_filters/logging_test.go new file mode 100644 index 000000000..d3d6303f7 --- /dev/null +++ b/modules/security/http_filters/logging_test.go @@ -0,0 +1,67 @@ +package http_filters + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + httprouter "infini.sh/framework/core/api/router" + "infini.sh/framework/core/env" + "infini.sh/framework/core/global" +) + +func TestLoggingFilterSkipsAccessLogWhenDisabled(t *testing.T) { + oldEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.PathConfig.Data = t.TempDir() + testEnv.SystemConfig.PathConfig.Log = t.TempDir() + testEnv.SystemConfig.WebAppConfig.AccessLog = false + global.RegisterEnv(testEnv) + defer global.RegisterEnv(oldEnv) + + accessLogHandler = nil + defer func() { accessLogHandler = nil }() + + filter := &LoggingFilter{} + handler := filter.ApplyFilter(http.MethodGet, "/hello", nil, func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodGet, "/hello", nil) + resp := httptest.NewRecorder() + handler(resp, req, nil) + + accessLogPath := filepath.Join(testEnv.GetLogDir(), "access.log") + if _, err := os.Stat(accessLogPath); !os.IsNotExist(err) { + t.Fatalf("expected access log file to be absent when disabled, stat err=%v", err) + } +} + +func TestLoggingFilterWritesAccessLogWhenEnabled(t *testing.T) { + oldEnv := global.Env() + testEnv := env.EmptyEnv() + testEnv.SystemConfig.PathConfig.Data = t.TempDir() + testEnv.SystemConfig.PathConfig.Log = t.TempDir() + testEnv.SystemConfig.WebAppConfig.AccessLog = true + global.RegisterEnv(testEnv) + defer global.RegisterEnv(oldEnv) + + accessLogHandler = nil + defer func() { accessLogHandler = nil }() + + filter := &LoggingFilter{} + handler := filter.ApplyFilter(http.MethodGet, "/hello", nil, func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + w.WriteHeader(http.StatusNoContent) + }) + + req := httptest.NewRequest(http.MethodGet, "/hello", nil) + resp := httptest.NewRecorder() + handler(resp, req, nil) + + accessLogPath := filepath.Join(testEnv.GetLogDir(), "access.log") + if _, err := os.Stat(accessLogPath); err != nil { + t.Fatalf("expected access log file to exist when enabled, got %v", err) + } +} From a2ccf877e2413811f35f8a97c847a855b66595f3 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 31 May 2026 12:06:37 +0800 Subject: [PATCH 094/137] improve: legence agent register for console --- modules/configs/client/client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index 8e0ce3d52..8d58c43e5 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -106,6 +106,8 @@ func ConnectToManager() error { panic(err) } global.Register(configRegisterEnvKey, true) + } else { + return fmt.Errorf("failed to register to config manager: status=%d, body=%s", res.StatusCode, strings.TrimSpace(string(res.Body))) } } else { log.Error("failed to register to config manager,", err, ",", server) From 04e3fad150901177838655a907ff35d186da6d2f Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 31 May 2026 12:38:03 +0800 Subject: [PATCH 095/137] improve: credention select manul first --- core/elastic/actions.go | 41 ++++++++++++++++++++++++------------ core/elastic/actions_test.go | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/core/elastic/actions.go b/core/elastic/actions.go index 3e562470b..9401a3a14 100644 --- a/core/elastic/actions.go +++ b/core/elastic/actions.go @@ -311,12 +311,9 @@ func (meta *ElasticsearchMetadata) GetActiveHost() string { for _, v := range hosts { if v != "" { if IsHostAvailable(v) { - //add to cache - info, ok := GetHostAvailableInfo(v) - if ok && info != nil { - if info.IsAvailable() { - meta.activeHost = info - } + info := meta.ensureAvailableHostInfo(v) + if info != nil && info.IsAvailable() { + meta.activeHost = info } return v @@ -331,12 +328,9 @@ func (meta *ElasticsearchMetadata) GetActiveHost() string { v := v1.GetHttpPublishHost() if v != "" { if IsHostAvailable(v) { - //add to cache - info, ok := GetHostAvailableInfo(v) - if ok && info != nil { - if info.IsAvailable() { - meta.activeHost = info - } + info := meta.ensureAvailableHostInfo(v) + if info != nil && info.IsAvailable() { + meta.activeHost = info } return v } @@ -369,12 +363,33 @@ func (meta *ElasticsearchMetadata) getAvailableSeedHost() (string, *NodeAvailabl if info, ok := GetHostAvailableInfo(host); ok && info != nil && info.IsAvailable() { return host, info } - return host, nil + return host, meta.ensureAvailableHostInfo(host) } return "", nil } +func (meta *ElasticsearchMetadata) ensureAvailableHostInfo(host string) *NodeAvailable { + if host == "" { + return nil + } + + host = util.UnifyLocalAddress(host) + if info, ok := GetHostAvailableInfo(host); ok && info != nil { + return info + } + + info := &NodeAvailable{ + Host: host, + ClusterID: meta.Config.ID, + available: true, + lastCheck: time.Now(), + lastSuccess: time.Now(), + } + hosts.Store(host, info) + return info +} + func (meta *ElasticsearchMetadata) IsTLS() bool { return meta.GetSchema() == "https" } diff --git a/core/elastic/actions_test.go b/core/elastic/actions_test.go index d558608b2..8b155fbe1 100644 --- a/core/elastic/actions_test.go +++ b/core/elastic/actions_test.go @@ -89,6 +89,44 @@ func TestGetActiveHostFallsBackToCachedDiscoveredHostWhenSeedUnavailable(t *test } } +func TestGetActiveHostInitializesAvailableSeedHostInfoFromAvailabilityCache(t *testing.T) { + const ( + clusterID = "seed-host-cache-init-cluster" + seedHost = "192.168.3.185:9220" + ) + + cfg := &ElasticsearchConfig{ + ORMObjectBase: orm.ORMObjectBase{ID: clusterID}, + Name: clusterID, + Host: seedHost, + Hosts: []string{seedHost}, + Enabled: true, + } + + meta := &ElasticsearchMetadata{Config: cfg} + nodeAvailCache.Put(seedHost, true) + hosts.Delete(seedHost) + t.Cleanup(func() { + hosts.Delete(seedHost) + }) + + got := meta.GetActiveHost() + if got != seedHost { + t.Fatalf("expected seed host %q, got %q", seedHost, got) + } + + info, ok := GetHostAvailableInfo(seedHost) + if !ok || info == nil { + t.Fatalf("expected host info for %q to be initialized", seedHost) + } + if !info.IsAvailable() { + t.Fatalf("expected host info for %q to be marked available", seedHost) + } + if info.ClusterID != clusterID { + t.Fatalf("expected cluster id %q, got %q", clusterID, info.ClusterID) + } +} + func TestShouldTraceUnavailableReasonForUnmonitoredCluster(t *testing.T) { meta := &ElasticsearchMetadata{ Config: &ElasticsearchConfig{ From c7153d3534d008b3faa39cf702622cb7b8dacf42 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 31 May 2026 18:43:11 +0800 Subject: [PATCH 096/137] improve: reduce debug log only for check --- core/security/validate.go | 2 +- modules/queue/disk_queue/consumer.go | 2 +- plugins/queue/consumer/consumer.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/security/validate.go b/core/security/validate.go index 0ae2ce4e0..f2fac46b7 100644 --- a/core/security/validate.go +++ b/core/security/validate.go @@ -85,7 +85,7 @@ func ValidateLogin(w http.ResponseWriter, r *http.Request) (session *UserSession if claims == nil || !claims.UserSessionInfo.IsValid() { claims, err = f(w, r) if claims != nil { - log.Debug("get valid auth info from: ", key) + log.Trace("get valid auth info from: ", key) return false } } diff --git a/modules/queue/disk_queue/consumer.go b/modules/queue/disk_queue/consumer.go index 4f4529ae3..cf0d14c75 100644 --- a/modules/queue/disk_queue/consumer.go +++ b/modules/queue/disk_queue/consumer.go @@ -245,7 +245,7 @@ READ_MSG: oldPart := d.segment Notify(d.queue, ReadComplete, d.segment) ctx.UpdateNextOffset(d.segment, d.readPos) //update next offset - log.Debugf("EOF, but current read segment_id [%v] is less than current write segment_id [%v], increase ++", oldPart, d.diskQueue.writeSegmentNum) + log.Tracef("EOF, but current read segment_id [%v] is less than current write segment_id [%v], increase ++", oldPart, d.diskQueue.writeSegmentNum) err = d.ResetOffset(d.segment+1, 0) //locate next segment if err != nil { if strings.Contains(err.Error(), "not found") { diff --git a/plugins/queue/consumer/consumer.go b/plugins/queue/consumer/consumer.go index 91ca5a9a9..9f9d2bc8f 100755 --- a/plugins/queue/consumer/consumer.go +++ b/plugins/queue/consumer/consumer.go @@ -233,7 +233,7 @@ func (processor *QueueConsumerProcessor) Process(c *pipeline.Context) error { } } } - log.Debug("exit consumer processor") + log.Trace("exit consumer processor") }() //handle updates @@ -256,7 +256,7 @@ func (processor *QueueConsumerProcessor) Process(c *pipeline.Context) error { } } processor.detectorRunning = false - log.Debug("exit detector for active queue") + log.Trace("exit detector for active queue") processor.wg.Done() }() @@ -307,7 +307,7 @@ func (processor *QueueConsumerProcessor) Process(c *pipeline.Context) error { log.Tracef("quite detect after idle for %v ms", processor.config.QuitDetectAfterIdleInMs) inflight := util.MapLength(&processor.inFlightQueueConfigs) if inflight == 0 { - log.Debugf("quite detect after idle for %v ms, inflight: %v", processor.config.QuitDetectAfterIdleInMs, inflight) + log.Tracef("quite detect after idle for %v ms, inflight: %v", processor.config.QuitDetectAfterIdleInMs, inflight) return } } From 19fe96fd6d45de3a072247f06b8b3cfb39e1217c Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 31 May 2026 22:33:11 +0800 Subject: [PATCH 097/137] fix: legence agent auth faile with register --- modules/configs/client/client.go | 23 +++++++++++++++++++++++ modules/configs/client/client_test.go | 19 ++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index 8d58c43e5..ec928159b 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -53,6 +53,7 @@ import ( const bucketName = "instance_registered" const configRegisterEnvKey = "CONFIG_MANAGED_SUCCESS" +const legacyManagedRegisterCompatMaxVersion = "1.30.4" var postRegisterHooks []func(server string, res *util.Result) error @@ -119,6 +120,9 @@ func buildManagedRegisterAccessToken(info model.Instance) (*common.RegisterToken if !common.SupportsManagedAccessToken(info.Application.Name) { return nil, nil } + if shouldSkipManagedRegisterAccessToken(info.Application.Version.VersionNumber) { + return nil, nil + } accessToken, err := common.EnsureTokenInKeystore(common.AgentAccessTokenKeystoreKey) if err != nil { return nil, err @@ -134,6 +138,25 @@ func buildManagedRegisterAccessToken(info model.Instance) (*common.RegisterToken }, nil } +func shouldSkipManagedRegisterAccessToken(version string) bool { + version = strings.TrimSpace(version) + if version == "" { + return false + } + parsed, err := util.ParseSemantic(version) + if err != nil { + parsed, err = util.ParseGeneric(version) + if err != nil { + return false + } + } + cmp, err := parsed.Compare(legacyManagedRegisterCompatMaxVersion) + if err != nil { + return false + } + return cmp <= 0 +} + func AddPostRegisterHook(hook func(server string, res *util.Result) error) { if hook != nil { postRegisterHooks = append(postRegisterHooks, hook) diff --git a/modules/configs/client/client_test.go b/modules/configs/client/client_test.go index af46963f5..f6d6c99f0 100644 --- a/modules/configs/client/client_test.go +++ b/modules/configs/client/client_test.go @@ -57,7 +57,10 @@ func TestBuildManagedRegisterAccessToken(t *testing.T) { instance := model.Instance{} instance.ID = "gateway-1" - instance.Application = env.Application{Name: "gateway"} + instance.Application = env.Application{ + Name: "gateway", + Version: env.Version{VersionNumber: "1.30.5"}, + } registerToken, err := buildManagedRegisterAccessToken(instance) if err != nil { @@ -80,6 +83,20 @@ func TestBuildManagedRegisterAccessToken(t *testing.T) { if registerToken != nil { t.Fatalf("expected no managed register token, got %#v", registerToken) } + + legacy := model.Instance{} + legacy.ID = "legacy-agent-1" + legacy.Application = env.Application{ + Name: "agent", + Version: env.Version{VersionNumber: "1.30.4"}, + } + registerToken, err = buildManagedRegisterAccessToken(legacy) + if err != nil { + t.Fatalf("expected nil error for legacy agent, got %v", err) + } + if registerToken != nil { + t.Fatalf("expected legacy agent to skip managed register token, got %#v", registerToken) + } } func TestListenConfigChangesStillSyncsAfterHTTPClientInit(t *testing.T) { From ee89304a1c0993fbc58fa91de005255626a77452 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 31 May 2026 22:58:37 +0800 Subject: [PATCH 098/137] fix: legence agent auth faile with register for log --- core/event/store.go | 4 +- modules/configs/client/client.go | 62 +++++++++++++++++++-- modules/configs/client/client_test.go | 46 +++++++++++++++ modules/elastic/adapter/elasticsearch/v0.go | 2 +- modules/elastic/adapter/elasticsearch/v7.go | 2 +- modules/elastic/adapter/elasticsearch/v8.go | 2 +- modules/elastic/orm.go | 2 +- 7 files changed, 109 insertions(+), 11 deletions(-) diff --git a/core/event/store.go b/core/event/store.go index e5634b778..ba64c488b 100644 --- a/core/event/store.go +++ b/core/event/store.go @@ -61,7 +61,7 @@ func SaveWithTimestamp(event *Event, time2 time.Time) error { } if global.Env().IsDebug { - log.Debugf("%v-%v: %v", event.Metadata.Category, event.Metadata.Name, string(util.MustToJSONBytes(event.Metadata))) + log.Tracef("%v-%v: %v", event.Metadata.Category, event.Metadata.Name, string(util.MustToJSONBytes(event.Metadata))) } event.Timestamp = time2 @@ -102,7 +102,7 @@ func SaveLog(event *Event) error { } if global.Env().IsDebug { - log.Debugf("%v-%v: %v, %v", event.Metadata.Category, event.Metadata.Name, util.MustToJSON(event.Metadata), util.MustToJSON(event.Fields)) + log.Tracef("%v-%v: %v, %v", event.Metadata.Category, event.Metadata.Name, util.MustToJSON(event.Metadata), util.MustToJSON(event.Fields)) } stats.Increment("metrics.savelog", event.Metadata.Category, event.Metadata.Name) diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index ec928159b..4c980839a 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -54,8 +54,21 @@ import ( const bucketName = "instance_registered" const configRegisterEnvKey = "CONFIG_MANAGED_SUCCESS" const legacyManagedRegisterCompatMaxVersion = "1.30.4" +const unauthorizedRegisterRetryInterval = 10 * time.Second var postRegisterHooks []func(server string, res *util.Result) error +var unauthorizedRegisterRetryLock sync.Mutex +var lastUnauthorizedRegisterRetryAt time.Time +var clearManagedRegistrationStateFunc = clearManagedRegistrationState +var reconnectToManagerFunc = func() error { return ConnectToManager() } + +func truncateManagerResponseBodyForLog(body []byte) string { + text := strings.TrimSpace(string(body)) + if len(text) <= 256 { + return text + } + return text[:256] + "...(truncated)" +} func ConnectToManager() error { cfg := global.Env().SystemConfig.Configs @@ -70,15 +83,14 @@ func ConnectToManager() error { if !cfg.AlwaysRegisterAfterRestart { if exists, err := kv.ExistsKey(bucketName, []byte(global.Env().SystemConfig.NodeConfig.ID)); exists && err == nil { //already registered skip further process - log.Info("already registered to config manager") + log.Infof("skip config manager registration for instance %v: local registration marker exists", global.Env().SystemConfig.NodeConfig.ID) global.Register(configRegisterEnvKey, true) return nil } } - log.Info("register new instance to config manager") - info := model.GetInstanceInfo() + log.Infof("start config manager registration for instance %v against %d server(s)", info.ID, len(cfg.Servers)) registerReq := common.InstanceRegisterRequest{ Client: info, } @@ -101,17 +113,18 @@ func ConnectToManager() error { if err := execPostRegisterHooks(server, res); err != nil { return err } - log.Infof("success register to config manager: %v", string(server)) + log.Infof("config manager registration succeeded for instance %v via %v: status=%d", info.ID, server, res.StatusCode) err := kv.AddValue(bucketName, []byte(global.Env().SystemConfig.NodeConfig.ID), []byte(util.GetLowPrecisionCurrentTime().String())) if err != nil { panic(err) } global.Register(configRegisterEnvKey, true) } else { + log.Warnf("config manager registration failed for instance %v via %v: status=%d, body=%s", info.ID, server, res.StatusCode, truncateManagerResponseBodyForLog(res.Body)) return fmt.Errorf("failed to register to config manager: status=%d, body=%s", res.StatusCode, strings.TrimSpace(string(res.Body))) } } else { - log.Error("failed to register to config manager,", err, ",", server) + log.Errorf("config manager registration request failed for instance %v via %v: %v", info.ID, server, err) } return err } @@ -163,6 +176,41 @@ func AddPostRegisterHook(hook func(server string, res *util.Result) error) { } } +func clearManagedRegistrationState() error { + global.Register(configRegisterEnvKey, false) + instanceID := strings.TrimSpace(global.Env().SystemConfig.NodeConfig.ID) + if instanceID == "" { + return nil + } + return kv.DeleteKey(bucketName, []byte(instanceID)) +} + +func handleUnauthorizedConfigSyncResponse(res *util.Result) bool { + if res == nil || res.StatusCode != http.StatusUnauthorized { + return false + } + + unauthorizedRegisterRetryLock.Lock() + if !lastUnauthorizedRegisterRetryAt.IsZero() && time.Since(lastUnauthorizedRegisterRetryAt) < unauthorizedRegisterRetryInterval { + unauthorizedRegisterRetryLock.Unlock() + return true + } + lastUnauthorizedRegisterRetryAt = time.Now() + unauthorizedRegisterRetryLock.Unlock() + + log.Warn("config sync unauthorized, clearing local registration state and retrying registration") + if err := clearManagedRegistrationStateFunc(); err != nil { + log.Warnf("failed to clear local registration state after unauthorized config sync: %v", err) + return true + } + if err := reconnectToManagerFunc(); err != nil { + log.Warnf("failed to re-register to config manager after unauthorized config sync: %v", err) + return true + } + log.Info("re-registered to config manager after unauthorized config sync") + return true +} + func execPostRegisterHooks(server string, res *util.Result) error { for _, hook := range postRegisterHooks { if err := hook(server, res); err != nil { @@ -278,6 +326,10 @@ func ListenConfigChanges() error { } if res != nil { + if handleUnauthorizedConfigSyncResponse(res) { + return + } + obj := common.ConfigSyncResponse{} err := util.FromJSONBytes(res.Body, &obj) if err != nil { diff --git a/modules/configs/client/client_test.go b/modules/configs/client/client_test.go index f6d6c99f0..79847814f 100644 --- a/modules/configs/client/client_test.go +++ b/modules/configs/client/client_test.go @@ -9,6 +9,7 @@ import ( "sync" "sync/atomic" "testing" + "time" "infini.sh/framework/core/config" "infini.sh/framework/core/env" @@ -158,3 +159,48 @@ func TestListenConfigChangesStillSyncsAfterHTTPClientInit(t *testing.T) { t.Fatalf("expected one immediate sync request, got %d", syncRequests.Load()) } } + +func TestHandleUnauthorizedConfigSyncResponseClearsStateAndReconnects(t *testing.T) { + oldClear := clearManagedRegistrationStateFunc + oldReconnect := reconnectToManagerFunc + oldRetryAt := lastUnauthorizedRegisterRetryAt + t.Cleanup(func() { + clearManagedRegistrationStateFunc = oldClear + reconnectToManagerFunc = oldReconnect + lastUnauthorizedRegisterRetryAt = oldRetryAt + }) + + var cleared atomic.Int32 + var reconnected atomic.Int32 + clearManagedRegistrationStateFunc = func() error { + cleared.Add(1) + return nil + } + reconnectToManagerFunc = func() error { + reconnected.Add(1) + return nil + } + lastUnauthorizedRegisterRetryAt = time.Time{} + + handled := handleUnauthorizedConfigSyncResponse(&util.Result{StatusCode: http.StatusUnauthorized}) + if !handled { + t.Fatal("expected unauthorized config sync response to be handled") + } + if cleared.Load() != 1 { + t.Fatalf("expected local registration state to be cleared once, got %d", cleared.Load()) + } + if reconnected.Load() != 1 { + t.Fatalf("expected reconnect to run once, got %d", reconnected.Load()) + } + + handled = handleUnauthorizedConfigSyncResponse(&util.Result{StatusCode: http.StatusUnauthorized}) + if !handled { + t.Fatal("expected throttled unauthorized config sync response to still be handled") + } + if cleared.Load() != 1 { + t.Fatalf("expected throttled retry not to clear state again, got %d", cleared.Load()) + } + if reconnected.Load() != 1 { + t.Fatalf("expected throttled retry not to reconnect again, got %d", reconnected.Load()) + } +} diff --git a/modules/elastic/adapter/elasticsearch/v0.go b/modules/elastic/adapter/elasticsearch/v0.go index 335d8215a..873ede19a 100755 --- a/modules/elastic/adapter/elasticsearch/v0.go +++ b/modules/elastic/adapter/elasticsearch/v0.go @@ -591,7 +591,7 @@ func (c *ESAPIV0) Search(indexName string, query *elastic.SearchRequest) (*elast js := query.ToJSONString() if global.Env().IsDebug { - log.Info(js) + log.Trace(js) } return c.SearchWithRawQueryDSL(indexName, util.UnsafeStringToBytes(js)) diff --git a/modules/elastic/adapter/elasticsearch/v7.go b/modules/elastic/adapter/elasticsearch/v7.go index 9b0c51c32..9648fbf70 100755 --- a/modules/elastic/adapter/elasticsearch/v7.go +++ b/modules/elastic/adapter/elasticsearch/v7.go @@ -374,7 +374,7 @@ func (c *ESAPIV7) Create(indexName, docType string, id interface{}, data interfa } if global.Env().IsDebug { - log.Debug("creating doc: ", url, ",", string(js)) + log.Trace("creating doc: ", url, ",", string(js)) } if err != nil { diff --git a/modules/elastic/adapter/elasticsearch/v8.go b/modules/elastic/adapter/elasticsearch/v8.go index c6f0dc0a4..427be5bea 100644 --- a/modules/elastic/adapter/elasticsearch/v8.go +++ b/modules/elastic/adapter/elasticsearch/v8.go @@ -280,7 +280,7 @@ func (c *ESAPIV8) Create(indexName, docType string, id interface{}, data interfa } if global.Env().IsDebug { - log.Debug("creating doc: ", url, ",", string(js)) + log.Trace("creating doc: ", url, ",", string(js)) } if err != nil { diff --git a/modules/elastic/orm.go b/modules/elastic/orm.go index 25127e6de..647d18c60 100755 --- a/modules/elastic/orm.go +++ b/modules/elastic/orm.go @@ -559,7 +559,7 @@ func (handler *ElasticORM) Search(t interface{}, q *api.Query) (error, api.Resul } if global.Env().IsDebug { - log.Info(util.MustToJSON(request)) + log.Trace(util.MustToJSON(request)) } searchResponse, err = handler.Client.Search(indexName, &request) From 0afba6dcb56a726600c6402d0b2917ac04ae63f7 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 1 Jun 2026 07:42:15 +0800 Subject: [PATCH 099/137] fix: atomic register at time --- modules/configs/client/client.go | 29 +++++++++++++++++++++------ modules/configs/client/client_test.go | 21 +++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index 4c980839a..cc62f5f0d 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -36,6 +36,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" log "github.com/cihub/seelog" @@ -61,6 +62,7 @@ var unauthorizedRegisterRetryLock sync.Mutex var lastUnauthorizedRegisterRetryAt time.Time var clearManagedRegistrationStateFunc = clearManagedRegistrationState var reconnectToManagerFunc = func() error { return ConnectToManager() } +var configSyncInProgress atomic.Bool func truncateManagerResponseBodyForLog(body []byte) string { text := strings.TrimSpace(string(body)) @@ -70,6 +72,14 @@ func truncateManagerResponseBodyForLog(body []byte) string { return text[:256] + "...(truncated)" } +func tryStartManagedConfigSync() bool { + return configSyncInProgress.CompareAndSwap(false, true) +} + +func finishManagedConfigSync() { + configSyncInProgress.Store(false) +} + func ConnectToManager() error { cfg := global.Env().SystemConfig.Configs if !cfg.Managed { @@ -296,15 +306,21 @@ func ListenConfigChanges() error { if global.Env().SystemConfig.Configs.Managed { initManagerHTTPClient() - //init config sync listening - req := common.ConfigSyncRequest{} - req.Client = model.GetInstanceInfo() - var syncFunc = func() { + if !tryStartManagedConfigSync() { + if global.Env().IsDebug { + log.Trace("skip overlapping config sync") + } + return + } + defer finishManagedConfigSync() + if global.Env().IsDebug { log.Trace("fetch configs from manger") } + req := common.ConfigSyncRequest{} + req.Client = model.GetInstanceInfo() cfgs := config.GetConfigs(false, false) req.Configs = cfgs req.Hash = util.MD5digestString(util.MustToJSONBytes(cfgs)) @@ -313,10 +329,11 @@ func ListenConfigChanges() error { request := util.Request{Method: util.Verb_POST} request.ContentType = "application/json" request.Path = common.SYNC_API - request.Body = util.MustToJSONBytes(req) + requestBody := util.MustToJSONBytes(req) + request.Body = requestBody if global.Env().IsDebug { - log.Debug("config sync request: ", string(util.MustToJSONBytes(req))) + log.Debug("config sync request: ", string(requestBody)) } _, res, err := DoManagerRequest(&request) diff --git a/modules/configs/client/client_test.go b/modules/configs/client/client_test.go index 79847814f..aeedc9a22 100644 --- a/modules/configs/client/client_test.go +++ b/modules/configs/client/client_test.go @@ -204,3 +204,24 @@ func TestHandleUnauthorizedConfigSyncResponseClearsStateAndReconnects(t *testing t.Fatalf("expected throttled retry not to reconnect again, got %d", reconnected.Load()) } } + +func TestManagedConfigSyncGuardPreventsOverlap(t *testing.T) { + configSyncInProgress.Store(false) + t.Cleanup(func() { + configSyncInProgress.Store(false) + }) + + if !tryStartManagedConfigSync() { + t.Fatal("expected first config sync to start") + } + if tryStartManagedConfigSync() { + t.Fatal("expected overlapping config sync to be rejected") + } + + finishManagedConfigSync() + + if !tryStartManagedConfigSync() { + t.Fatal("expected config sync to start again after previous one finished") + } + finishManagedConfigSync() +} From 3aae76bafcedc8684393a9005572f44a065fea39 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 1 Jun 2026 12:16:11 +0800 Subject: [PATCH 100/137] fix: challenge same --- core/security/service_registry.go | 2 +- modules/security/rbac/account_login.go | 34 ++++++++++++++------------ 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/core/security/service_registry.go b/core/security/service_registry.go index 494c4ec3f..f1d8ec2bb 100644 --- a/core/security/service_registry.go +++ b/core/security/service_registry.go @@ -109,7 +109,7 @@ func GetUserByLogin(login string) (bool, *UserAccount, error) { return false, nil, errors.New("no AuthenticationBackend was found") } - return false, nil, errors.New("not found") + return false, nil, nil } // AuthenticateAccountPasswordLogin tries application-provided password login providers diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go index be79791ee..792b7f4df 100644 --- a/modules/security/rbac/account_login.go +++ b/modules/security/rbac/account_login.go @@ -184,27 +184,29 @@ func (req accountLoginRequest) NormalizedLogin() string { return "" } -// buildLoginChallengeResponse keeps the challenge negotiation explicit: challenge-capable -// accounts get the proof derivation inputs, while older accounts stay on plain login. +// buildLoginChallengeResponse always returns a challenge-format response to prevent user +// enumeration: callers cannot distinguish an existing account from a non-existent one by +// observing the response shape. For accounts that do not exist or have not yet derived +// challenge material, a throwaway salt is generated so the client goes through the full +// proof-derivation flow; the proof will be rejected by the Login handler with a generic +// "invalid login or password" error. func buildLoginChallengeResponse(login string, exists bool, user *security.UserAccount) util.MapStr { + salt := util.GenerateSecureString(32) if exists && security.CanUsePasswordChallenge(user) { - // The challenge payload gives clients everything needed to derive a proof - // locally without sending the raw password back to the server. - challenge := security.NewLoginChallenge(login) - return util.MapStr{ - "status": "ok", - "method": security.PasswordChallengeMethod, - "algorithm": security.PasswordChallengeAlgorithm, - "iterations": security.PasswordChallengeIterations, - "challenge_id": challenge.ID, - "nonce": challenge.Nonce, - "salt": user.PasswordSalt, - } + salt = user.PasswordSalt } + // The challenge payload gives clients everything needed to derive a proof + // locally without sending the raw password back to the server. + challenge := security.NewLoginChallenge(login) return util.MapStr{ - "status": "ok", - "method": "plain", + "status": "ok", + "method": security.PasswordChallengeMethod, + "algorithm": security.PasswordChallengeAlgorithm, + "iterations": security.PasswordChallengeIterations, + "challenge_id": challenge.ID, + "nonce": challenge.Nonce, + "salt": salt, } } From 66429422ce375db2adb6cbff661928a5c2f918b6 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 1 Jun 2026 16:40:14 +0800 Subject: [PATCH 101/137] fix: ui work well with test and log mask --- modules/configs/client/client.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index cc62f5f0d..d516cde06 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -64,6 +64,29 @@ var clearManagedRegistrationStateFunc = clearManagedRegistrationState var reconnectToManagerFunc = func() error { return ConnectToManager() } var configSyncInProgress atomic.Bool +// maskURLInError replaces http(s):// URLs in error messages to avoid leaking internal addresses in logs. +func maskURLInError(err error) string { + if err == nil { + return "" + } + msg := err.Error() + for _, scheme := range []string{"https://", "http://"} { + for { + idx := strings.Index(msg, scheme) + if idx < 0 { + break + } + end := strings.IndexAny(msg[idx:], " \"'\n\t") + if end < 0 { + msg = msg[:idx] + "***" + break + } + msg = msg[:idx] + "***" + msg[idx+end:] + } + } + return msg +} + func truncateManagerResponseBodyForLog(body []byte) string { text := strings.TrimSpace(string(body)) if len(text) <= 256 { @@ -338,7 +361,7 @@ func ListenConfigChanges() error { _, res, err := DoManagerRequest(&request) if err != nil { - log.Error("failed to submit request to config manager,", err) + log.Error("failed to submit request to config manager,", maskURLInError(err)) return } From 9e18ac6e08e4aa9d7ebdd681337222d9b0a0f0a3 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 2 Jun 2026 10:55:20 +0800 Subject: [PATCH 102/137] improve: some page optimize and legacy login --- modules/queue/disk_queue/diskqueue.go | 17 ++++ modules/queue/disk_queue/diskqueue_test.go | 105 ++++++++++++++++++++ modules/security/rbac/account_login.go | 16 +-- modules/security/rbac/account_login_test.go | 17 ++++ 4 files changed, 149 insertions(+), 6 deletions(-) diff --git a/modules/queue/disk_queue/diskqueue.go b/modules/queue/disk_queue/diskqueue.go index 37e05c0a4..3ad1ce33e 100644 --- a/modules/queue/disk_queue/diskqueue.go +++ b/modules/queue/disk_queue/diskqueue.go @@ -303,6 +303,13 @@ func (d *DiskBasedQueue) Delete() error { return d.exit(true) } +func (d *DiskBasedQueue) ensureDataPathExists() error { + if d == nil || d.dataPath == "" { + return nil + } + return os.MkdirAll(d.dataPath, 0o755) +} + func (d *DiskBasedQueue) exit(deleted bool) error { d.Lock() @@ -570,6 +577,11 @@ func (d *DiskBasedQueue) writeOne(data []byte) WriteResponse { var res WriteResponse if d.writeFile == nil { + err = d.ensureDataPathExists() + if err != nil { + res.Error = err + return res + } curFileName := d.GetFileName(d.writeSegmentNum) d.writeFile, err = os.OpenFile(curFileName, os.O_RDWR|os.O_CREATE, 0600) if err != nil { @@ -892,6 +904,11 @@ func (d *DiskBasedQueue) persistMetaData() error { var f *os.File var err error + err = d.ensureDataPathExists() + if err != nil { + return err + } + fileName := d.metaDataFileName() tmpFileName := fmt.Sprintf("%s.%d.tmp", fileName, rand.Int()) diff --git a/modules/queue/disk_queue/diskqueue_test.go b/modules/queue/disk_queue/diskqueue_test.go index b38d59ef3..d041b8ab5 100644 --- a/modules/queue/disk_queue/diskqueue_test.go +++ b/modules/queue/disk_queue/diskqueue_test.go @@ -208,6 +208,111 @@ func TestRepairTailMetadataTruncatesIncompleteTailOnStartup(t *testing.T) { } } +func TestQueueRecreatesDataPathAfterDirectoryDeletion(t *testing.T) { + env1 := EmptyEnv() + env1.SystemConfig.PathConfig.Data = t.TempDir() + global.RegisterEnv(env1) + + queueName := "recreate-data-path" + cfg := &DiskQueueConfig{ + MinMsgSize: 1, + MaxMsgSize: 1024, + MaxBytesPerFile: 1024 * 1024, + } + normalizeDiskQueueConfig(cfg) + + dataPath := GetDataPath(queueName) + if err := os.MkdirAll(dataPath, 0o755); err != nil { + t.Fatalf("failed to create queue data dir: %v", err) + } + + dq := &DiskBasedQueue{ + name: queueName, + dataPath: dataPath, + cfg: cfg, + writePos: 0, + writeFile: nil, + } + + if err := os.RemoveAll(dataPath); err != nil { + t.Fatalf("failed to delete queue data dir: %v", err) + } + + if err := dq.sync(); err != nil { + t.Fatalf("expected sync to recreate deleted queue dir, got %v", err) + } + if _, err := os.Stat(dataPath); err != nil { + t.Fatalf("expected queue data dir to be recreated, got %v", err) + } + + res := dq.writeOne([]byte("hello")) + if res.Error != nil { + t.Fatalf("expected write to succeed after dir recreation, got %v", res.Error) + } + if _, err := os.Stat(dq.metaDataFileName()); err != nil { + t.Fatalf("expected metadata file to be recreated, got %v", err) + } + if _, err := os.Stat(dq.GetFileName(0)); err != nil { + t.Fatalf("expected segment file to be recreated, got %v", err) + } + + if dq.writeFile != nil { + _ = dq.writeFile.Close() + } +} + +func TestCloseSucceedsAfterQueueDirectoryDeletion(t *testing.T) { + env1 := EmptyEnv() + env1.SystemConfig.PathConfig.Data = t.TempDir() + global.RegisterEnv(env1) + + queueName := "close-after-delete" + cfg := &DiskQueueConfig{ + MinMsgSize: 1, + MaxMsgSize: 1024, + MaxBytesPerFile: 1024 * 1024, + SyncEveryRecords: 1 << 20, + SyncTimeoutInMS: 1 << 20, + ReadChanBuffer: 0, + WriteChanBuffer: 1, + } + normalizeDiskQueueConfig(cfg) + + dataPath := GetDataPath(queueName) + if err := os.MkdirAll(dataPath, 0o755); err != nil { + t.Fatalf("failed to create queue data dir: %v", err) + } + + dq := &DiskBasedQueue{ + name: queueName, + dataPath: dataPath, + cfg: cfg, + readChan: make(chan []byte, cfg.ReadChanBuffer), + depthChan: make(chan int64), + writeChan: make(chan []byte, cfg.WriteChanBuffer), + writeResponseChan: make(chan WriteResponse), + emptyChan: make(chan int), + emptyResponseChan: make(chan error), + exitChan: make(chan int), + exitSyncChan: make(chan int, 1), + consumersInReading: sync.Map{}, + } + go dq.ioLoop() + + res := dq.Put([]byte("hello")) + if res.Error != nil { + t.Fatalf("failed to put queue message: %v", res.Error) + } + + if err := os.RemoveAll(dataPath); err != nil { + t.Fatalf("failed to delete queue data dir: %v", err) + } + + if err := dq.Close(); err != nil { + t.Fatalf("expected close to succeed after queue dir deletion, got %v", err) + } +} + func TestResetOffsetSkipsMissingSegmentsUpToCurrentWriteSegment(t *testing.T) { env1 := EmptyEnv() env1.SystemConfig.PathConfig.Data = t.TempDir() diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go index 792b7f4df..80a847194 100644 --- a/modules/security/rbac/account_login.go +++ b/modules/security/rbac/account_login.go @@ -184,13 +184,17 @@ func (req accountLoginRequest) NormalizedLogin() string { return "" } -// buildLoginChallengeResponse always returns a challenge-format response to prevent user -// enumeration: callers cannot distinguish an existing account from a non-existent one by -// observing the response shape. For accounts that do not exist or have not yet derived -// challenge material, a throwaway salt is generated so the client goes through the full -// proof-derivation flow; the proof will be rejected by the Login handler with a generic -// "invalid login or password" error. +// buildLoginChallengeResponse returns plain login for existing legacy accounts that have +// not been upgraded with challenge material yet. Accounts that do not exist still receive +// a fake challenge payload to avoid user enumeration. func buildLoginChallengeResponse(login string, exists bool, user *security.UserAccount) util.MapStr { + if exists && !security.CanUsePasswordChallenge(user) { + return util.MapStr{ + "status": "ok", + "method": "plain", + } + } + salt := util.GenerateSecureString(32) if exists && security.CanUsePasswordChallenge(user) { salt = user.PasswordSalt diff --git a/modules/security/rbac/account_login_test.go b/modules/security/rbac/account_login_test.go index c9efbf75b..d2cde6608 100644 --- a/modules/security/rbac/account_login_test.go +++ b/modules/security/rbac/account_login_test.go @@ -207,6 +207,23 @@ func TestBuildLoginChallengeResponseFallsBackToPlain(t *testing.T) { } } +func TestBuildLoginChallengeResponseFakesChallengeForMissingUser(t *testing.T) { + resp := buildLoginChallengeResponse("missing@example.org", false, nil) + + if got := resp["method"]; got != security.PasswordChallengeMethod { + t.Fatalf("expected fake challenge method for missing user, got %v", got) + } + if resp["challenge_id"] == "" { + t.Fatal("expected fake challenge id for missing user") + } + if resp["nonce"] == "" { + t.Fatal("expected fake nonce for missing user") + } + if resp["salt"] == "" { + t.Fatal("expected fake salt for missing user") + } +} + // Upgraded accounts should return the exact challenge inputs the client needs next. func TestBuildLoginChallengeResponseReturnsChallenge(t *testing.T) { user := &security.UserAccount{Email: "admin@example.org"} From 7395d8b4560fdda67a38f6d43ea48513482dcb81 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 2 Jun 2026 11:57:50 +0800 Subject: [PATCH 103/137] improve: store no panic and pause only running --- core/event/store.go | 65 ++++++++++++++++++++++++++++++----- core/event/store_test.go | 74 ++++++++++++++++++++++++++++++++++++++++ core/pipeline/context.go | 4 ++- 3 files changed, 134 insertions(+), 9 deletions(-) create mode 100644 core/event/store_test.go diff --git a/core/event/store.go b/core/event/store.go index ba64c488b..2e7c34ab9 100644 --- a/core/event/store.go +++ b/core/event/store.go @@ -36,6 +36,56 @@ import ( "infini.sh/framework/core/util" ) +var pushQueueMessage = queue.Push +var getOrInitQueueConfig = queue.GetOrInitConfig + +func normalizeLabelValue(value interface{}) interface{} { + switch v := value.(type) { + case util.MapStr: + if len(v) == 1 { + if inner, ok := v["value"]; ok { + return normalizeLabelValue(inner) + } + if inner, ok := v["terms"]; ok { + return normalizeLabelValue(inner) + } + } + for key, item := range v { + v[key] = normalizeLabelValue(item) + } + return v + case map[string]interface{}: + if len(v) == 1 { + if inner, ok := v["value"]; ok { + return normalizeLabelValue(inner) + } + if inner, ok := v["terms"]; ok { + return normalizeLabelValue(inner) + } + } + for key, item := range v { + v[key] = normalizeLabelValue(item) + } + return v + case []interface{}: + for i, item := range v { + v[i] = normalizeLabelValue(item) + } + return v + default: + return value + } +} + +func normalizeEventLabels(event *Event) { + if event == nil || event.Metadata.Labels == nil { + return + } + for key, value := range event.Metadata.Labels { + event.Metadata.Labels[key] = normalizeLabelValue(value) + } +} + var meta *AgentMeta func RegisterMeta(m *AgentMeta) { @@ -60,6 +110,8 @@ func SaveWithTimestamp(event *Event, time2 time.Time) error { panic("event can't be nil") } + normalizeEventLabels(event) + if global.Env().IsDebug { log.Tracef("%v-%v: %v", event.Metadata.Category, event.Metadata.Name, string(util.MustToJSONBytes(event.Metadata))) } @@ -68,7 +120,7 @@ func SaveWithTimestamp(event *Event, time2 time.Time) error { //check event specified queue name if event.QueueName != "" { - return queue.Push(queue.GetOrInitConfig(event.QueueName), util.MustToJSONBytes(event)) + return pushQueueMessage(getOrInitQueueConfig(event.QueueName), util.MustToJSONBytes(event)) } else { //check default queue name if getMeta().DefaultMetricQueueName == "" { @@ -82,7 +134,7 @@ func SaveWithTimestamp(event *Event, time2 time.Time) error { } stats.Increment("metrics.save", event.Metadata.Category, event.Metadata.Name) - return queue.Push(queue.GetOrInitConfig(event.QueueName), util.MustToJSONBytes(event)) + return pushQueueMessage(getOrInitQueueConfig(event.QueueName), util.MustToJSONBytes(event)) } func Save(event *Event) error { @@ -94,6 +146,8 @@ func SaveLog(event *Event) error { panic("event can't be nil") } + normalizeEventLabels(event) + event.Timestamp = time.Now() event.Agent = getMeta() @@ -107,10 +161,5 @@ func SaveLog(event *Event) error { stats.Increment("metrics.savelog", event.Metadata.Category, event.Metadata.Name) - err := queue.Push(queue.GetOrInitConfig(getMeta().LoggingQueueName), util.MustToJSONBytes(event)) - if err != nil { - panic(err) - } - - return nil + return pushQueueMessage(getOrInitQueueConfig(getMeta().LoggingQueueName), util.MustToJSONBytes(event)) } diff --git a/core/event/store_test.go b/core/event/store_test.go new file mode 100644 index 000000000..add636993 --- /dev/null +++ b/core/event/store_test.go @@ -0,0 +1,74 @@ +package event + +import ( + "errors" + "testing" + + "infini.sh/framework/core/queue" + "infini.sh/framework/core/util" +) + +func TestNormalizeEventLabelsFlattensDSLWrappers(t *testing.T) { + item := &Event{ + Metadata: EventMetadata{ + Labels: util.MapStr{ + "cluster_id": util.MapStr{ + "terms": "infini_default_system_cluster", + }, + "cluster_uuid": map[string]interface{}{ + "value": "cluster-uuid", + }, + "roles": []interface{}{ + util.MapStr{"value": "data"}, + map[string]interface{}{"terms": "ingest"}, + }, + }, + }, + } + + normalizeEventLabels(item) + + if got := item.Metadata.Labels["cluster_id"]; got != "infini_default_system_cluster" { + t.Fatalf("expected flattened cluster_id label, got %#v", got) + } + if got := item.Metadata.Labels["cluster_uuid"]; got != "cluster-uuid" { + t.Fatalf("expected flattened cluster_uuid label, got %#v", got) + } + + roles, ok := item.Metadata.Labels["roles"].([]interface{}) + if !ok { + t.Fatalf("expected roles to remain a slice, got %#v", item.Metadata.Labels["roles"]) + } + if len(roles) != 2 || roles[0] != "data" || roles[1] != "ingest" { + t.Fatalf("expected flattened roles entries, got %#v", roles) + } +} + +func TestSaveLogReturnsQueueError(t *testing.T) { + originalPush := pushQueueMessage + originalGetOrInitQueueConfig := getOrInitQueueConfig + originalMeta := meta + t.Cleanup(func() { + pushQueueMessage = originalPush + getOrInitQueueConfig = originalGetOrInitQueueConfig + meta = originalMeta + }) + + pushQueueMessage = func(_ *queue.QueueConfig, _ []byte) error { + return errors.New("readonly") + } + getOrInitQueueConfig = func(_ string) *queue.QueueConfig { + return &queue.QueueConfig{} + } + meta = &AgentMeta{LoggingQueueName: "logging"} + + err := SaveLog(&Event{ + Metadata: EventMetadata{ + Category: "task", + Name: "logging", + }, + }) + if err == nil || err.Error() != "readonly" { + t.Fatalf("expected readonly error, got %v", err) + } +} diff --git a/core/pipeline/context.go b/core/pipeline/context.go index 8936789e4..d5b6453e0 100755 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -446,5 +446,7 @@ func (ctx *Context) pushPipelineLog() { }, } - event.SaveLog(&eventData) + if err := event.SaveLog(&eventData); err != nil { + log.Errorf("failed to save pipeline log event, pipeline: %s, context: %s, err: %v", ctx.Config.Name, ctx.id, err) + } } From 9e9aef5ee80f9f993f2b0af9df2e44813fe3ffb3 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 2 Jun 2026 14:34:46 +0800 Subject: [PATCH 104/137] fix: migration can't stop with arm64 --- core/queue/api.go | 54 ++++++---- core/queue/api_test.go | 228 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 260 insertions(+), 22 deletions(-) create mode 100644 core/queue/api_test.go diff --git a/core/queue/api.go b/core/queue/api.go index e35182c8c..3740c045f 100755 --- a/core/queue/api.go +++ b/core/queue/api.go @@ -142,39 +142,49 @@ func AcquireConsumer(k *QueueConfig, consumer *ConsumerConfig, clientID string) panic(errors.New("clientID can't be nil")) } - //check if the consumer is in fighting list - if v, ok := consumersInFighting.Load(k.ID + consumer.Key()); ok { - if v != clientID { - //check the last touch time + fightingKey := k.ID + consumer.Key() + + for { + reserved := false + currentOwner, loaded := consumersInFighting.LoadOrStore(fightingKey, clientID) + if loaded && currentOwner != clientID { if consumer.ConsumeTimeoutInSeconds > 0 { t := consumer.GetLastActiveTime() if t != nil && int(time.Since(*t).Seconds()) > consumer.ConsumeTimeoutInSeconds { - consumersInFighting.Delete(k.ID + consumer.Key()) - stats.Increment("consumer", k.ID, consumer.GetID(), "expired") - //the consumer is in fighting and is already timeout - return nil, errors.Errorf("consumer:%v is already in fighting list, but expired in: %v, remove it from the fighting list", consumer.Key(), time.Since(*t).Seconds()) + if consumersInFighting.CompareAndDelete(fightingKey, currentOwner) { + stats.Increment("consumer", k.ID, consumer.GetID(), "expired") + } + continue } } stats.Increment("consumer", k.ID, consumer.GetID(), "contend") - //the consumer is in fighting list and the clientID is not the same return nil, errors.New("the consumer is in fighting list") } - } - - handler := getAdvancedHandler(k) - if handler != nil { - v1, err := handler.AcquireConsumer(k, consumer) - if err != nil { - stats.Increment("consumer", k.ID, consumer.GetID(), "error_on_acquire") - return nil, err + if !loaded { + reserved = true } - //add the consumer to the fighting list - consumersInFighting.Store(k.ID+consumer.Key(), clientID) - stats.Increment("consumer", k.ID, consumer.GetID(), "acquired") - return v1, nil + handler := getAdvancedHandler(k) + if handler != nil { + acquired := false + defer func() { + if !acquired && reserved { + consumersInFighting.CompareAndDelete(fightingKey, clientID) + } + }() + + v1, err := handler.AcquireConsumer(k, consumer) + if err != nil { + stats.Increment("consumer", k.ID, consumer.GetID(), "error_on_acquire") + return nil, err + } + + acquired = true + stats.Increment("consumer", k.ID, consumer.GetID(), "acquired") + return v1, nil + } + panic(errors.New("handler is not registered")) } - panic(errors.New("handler is not registered")) } func ReleaseConsumer(k *QueueConfig, c *ConsumerConfig, consumer ConsumerAPI) error { diff --git a/core/queue/api_test.go b/core/queue/api_test.go new file mode 100644 index 000000000..829a2834a --- /dev/null +++ b/core/queue/api_test.go @@ -0,0 +1,228 @@ +package queue + +import ( + "errors" + "infini.sh/framework/core/stats" + "sync" + "testing" + "time" +) + +type acquireConsumerTestHandler struct { + acquireFunc func(k *QueueConfig, consumer *ConsumerConfig) (ConsumerAPI, error) +} + +func (h *acquireConsumerTestHandler) Name() string { return "test" } +func (h *acquireConsumerTestHandler) Init(string) error { return nil } +func (h *acquireConsumerTestHandler) Close(string) error { return nil } +func (h *acquireConsumerTestHandler) GetStorageSize(string) uint64 { return 0 } +func (h *acquireConsumerTestHandler) Destroy(string) error { return nil } +func (h *acquireConsumerTestHandler) GetQueues() []string { return nil } +func (h *acquireConsumerTestHandler) Push(string, []byte) error { return nil } +func (h *acquireConsumerTestHandler) LatestOffset(*QueueConfig) Offset { return Offset{} } +func (h *acquireConsumerTestHandler) GetOffset(*QueueConfig, *ConsumerConfig) (Offset, error) { + return Offset{}, nil +} +func (h *acquireConsumerTestHandler) DeleteOffset(*QueueConfig, *ConsumerConfig) error { return nil } +func (h *acquireConsumerTestHandler) CommitOffset(*QueueConfig, *ConsumerConfig, Offset) (bool, error) { + return true, nil +} +func (h *acquireConsumerTestHandler) AcquireConsumer(k *QueueConfig, consumer *ConsumerConfig) (ConsumerAPI, error) { + if h.acquireFunc != nil { + return h.acquireFunc(k, consumer) + } + return &acquireConsumerTestConsumer{}, nil +} +func (h *acquireConsumerTestHandler) ReleaseConsumer(*QueueConfig, *ConsumerConfig, ConsumerAPI) error { + return nil +} +func (h *acquireConsumerTestHandler) AcquireProducer(*QueueConfig) (ProducerAPI, error) { + return nil, nil +} +func (h *acquireConsumerTestHandler) ReleaseProducer(*QueueConfig, ProducerAPI) error { return nil } + +type acquireConsumerTestConsumer struct{} + +func (c *acquireConsumerTestConsumer) Close() error { return nil } +func (c *acquireConsumerTestConsumer) ResetOffset(int64, int64) error { return nil } +func (c *acquireConsumerTestConsumer) FetchMessages(*Context, int) ([]Message, bool, error) { + return nil, false, nil +} +func (c *acquireConsumerTestConsumer) CommitOffset(Offset) error { return nil } + +type acquireConsumerTestStats struct { + mu sync.Mutex + timestamps map[string]time.Time +} + +func (s *acquireConsumerTestStats) Increment(string, string) {} +func (s *acquireConsumerTestStats) IncrementBy(string, string, int64) {} +func (s *acquireConsumerTestStats) Decrement(string, string) {} +func (s *acquireConsumerTestStats) DecrementBy(string, string, int64) {} +func (s *acquireConsumerTestStats) Absolute(string, string, int64) {} +func (s *acquireConsumerTestStats) Timing(string, string, int64) {} +func (s *acquireConsumerTestStats) Gauge(string, string, int64) {} +func (s *acquireConsumerTestStats) Stat(string, string) int64 { return 0 } +func (s *acquireConsumerTestStats) StatsAll() string { return "" } +func (s *acquireConsumerTestStats) RecordTimestamp(category, key string, value time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + s.timestamps[category+"."+key] = value +} +func (s *acquireConsumerTestStats) GetTimestamp(category, key string) (time.Time, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.timestamps[category+"."+key] + if !ok { + return time.Time{}, errors.New("not found") + } + return v, nil +} +func (s *acquireConsumerTestStats) reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.timestamps = map[string]time.Time{} +} + +var acquireConsumerStatsOnce sync.Once +var acquireConsumerStatsHandler = &acquireConsumerTestStats{timestamps: map[string]time.Time{}} + +func withTestQueueHandler(t *testing.T, handler AdvancedQueueAPI) { + t.Helper() + previousDefaultHandler := defaultHandler + previousConsumersInFighting := consumersInFighting + acquireConsumerStatsOnce.Do(func() { + stats.Register(acquireConsumerStatsHandler) + }) + acquireConsumerStatsHandler.reset() + defaultHandler = handler + consumersInFighting = syncMapZero() + t.Cleanup(func() { + defaultHandler = previousDefaultHandler + consumersInFighting = previousConsumersInFighting + }) +} + +func syncMapZero() sync.Map { + return sync.Map{} +} + +func TestAcquireConsumerStoresReservation(t *testing.T) { + withTestQueueHandler(t, &acquireConsumerTestHandler{}) + + q := &QueueConfig{ID: "queue-1", Name: "queue-1"} + c := &ConsumerConfig{Group: "group", Name: "consumer"} + c.ID = "consumer-1" + + instance, err := AcquireConsumer(q, c, "client-1") + if err != nil { + t.Fatalf("expected acquire to succeed, got %v", err) + } + if instance == nil { + t.Fatal("expected consumer instance to be returned") + } + if owner, ok := consumersInFighting.Load(q.ID + c.Key()); !ok || owner != "client-1" { + t.Fatalf("expected fighting list reservation to be stored, got owner=%v exists=%v", owner, ok) + } +} + +func TestAcquireConsumerRollsBackReservationOnError(t *testing.T) { + withTestQueueHandler(t, &acquireConsumerTestHandler{ + acquireFunc: func(k *QueueConfig, consumer *ConsumerConfig) (ConsumerAPI, error) { + return nil, errors.New("boom") + }, + }) + + q := &QueueConfig{ID: "queue-1", Name: "queue-1"} + c := &ConsumerConfig{Group: "group", Name: "consumer"} + c.ID = "consumer-1" + + _, err := AcquireConsumer(q, c, "client-1") + if err == nil { + t.Fatal("expected acquire to fail") + } + if _, ok := consumersInFighting.Load(q.ID + c.Key()); ok { + t.Fatal("expected fighting list reservation to be rolled back") + } +} + +func TestAcquireConsumerRollsBackReservationOnPanic(t *testing.T) { + withTestQueueHandler(t, &acquireConsumerTestHandler{ + acquireFunc: func(k *QueueConfig, consumer *ConsumerConfig) (ConsumerAPI, error) { + panic("boom") + }, + }) + + q := &QueueConfig{ID: "queue-1", Name: "queue-1"} + c := &ConsumerConfig{Group: "group", Name: "consumer"} + c.ID = "consumer-1" + + defer func() { + if r := recover(); r == nil { + t.Fatal("expected acquire to panic") + } + if _, ok := consumersInFighting.Load(q.ID + c.Key()); ok { + t.Fatal("expected fighting list reservation to be rolled back after panic") + } + }() + + _, _ = AcquireConsumer(q, c, "client-1") +} + +func TestAcquireConsumerBlocksCompetingClient(t *testing.T) { + withTestQueueHandler(t, &acquireConsumerTestHandler{}) + + q := &QueueConfig{ID: "queue-1", Name: "queue-1"} + c := &ConsumerConfig{Group: "group", Name: "consumer"} + c.ID = "consumer-1" + consumersInFighting.Store(q.ID+c.Key(), "client-1") + + _, err := AcquireConsumer(q, c, "client-2") + if err == nil || err.Error() != "the consumer is in fighting list" { + t.Fatalf("expected fighting list error, got %v", err) + } +} + +func TestAcquireConsumerAllowsSameClientReentry(t *testing.T) { + withTestQueueHandler(t, &acquireConsumerTestHandler{}) + + q := &QueueConfig{ID: "queue-1", Name: "queue-1"} + c := &ConsumerConfig{Group: "group", Name: "consumer"} + c.ID = "consumer-1" + consumersInFighting.Store(q.ID+c.Key(), "client-1") + + instance, err := AcquireConsumer(q, c, "client-1") + if err != nil { + t.Fatalf("expected same client to re-enter, got %v", err) + } + if instance == nil { + t.Fatal("expected consumer instance for same-client reentry") + } +} + +func TestAcquireConsumerRetriesExpiredReservation(t *testing.T) { + withTestQueueHandler(t, &acquireConsumerTestHandler{}) + + q := &QueueConfig{ID: "queue-1", Name: "queue-1"} + c := &ConsumerConfig{ + Group: "group", + Name: "consumer", + ConsumeTimeoutInSeconds: 1, + } + c.ID = "consumer-1" + c.KeepActive() + stale := time.Now().Add(-3 * time.Second) + stats.Timestamp("consumer", c.ID+".last_active", stale) + consumersInFighting.Store(q.ID+c.Key(), "client-2") + + instance, err := AcquireConsumer(q, c, "client-1") + if err != nil { + t.Fatalf("expected expired reservation to be retried, got %v", err) + } + if instance == nil { + t.Fatal("expected consumer instance after expired reservation cleanup") + } + if owner, ok := consumersInFighting.Load(q.ID + c.Key()); !ok || owner != "client-1" { + t.Fatalf("expected ownership to move to client-1, got owner=%v exists=%v", owner, ok) + } +} From 144c33a079f20570c66581155083fb61966fdf0c Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 2 Jun 2026 16:38:03 +0800 Subject: [PATCH 105/137] improve: user login upgrade by challege --- modules/security/rbac/account_login.go | 19 +++++- modules/security/rbac/account_login_test.go | 64 +++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go index 80a847194..39db70d80 100644 --- a/modules/security/rbac/account_login.go +++ b/modules/security/rbac/account_login.go @@ -38,6 +38,10 @@ import ( "infini.sh/framework/core/util" ) +var persistPasswordChallengeUpgrade = func(ctx *orm.Context, user *security.UserAccount) error { + return orm.Save(ctx, user) +} + var ( // Keep the password and challenge paths aligned on one user-facing failure message. errInvalidLoginCredentials = errors.New("invalid login or password") @@ -163,7 +167,7 @@ func Login(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { } if !usedChallenge && nativeUser != nil { - upgradePasswordChallenge(nativeUser, req.Password) + upgradePasswordChallenge(nativeUser, login, req.Password) } if err, token := security.AddUserToSession(w, r, sessionUser); err != nil { @@ -272,7 +276,7 @@ func validateReplayNonce(r *http.Request, required bool) error { // upgradePasswordChallenge backfills verifier material after a successful legacy login so // existing native accounts can move onto the challenge flow without an offline migration. -func upgradePasswordChallenge(user *security.UserAccount, password string) { +func upgradePasswordChallenge(user *security.UserAccount, login, password string) { if user == nil || password == "" || security.CanUsePasswordChallenge(user) { return } @@ -286,9 +290,18 @@ func upgradePasswordChallenge(user *security.UserAccount, password string) { // logins can move onto the challenge flow without an explicit migration step. // This upgrade is best-effort; the current login already succeeded, so it should // not wait for an index refresh before returning to the caller. + if user.ID == "" { + userLogin := strings.TrimSpace(user.Email) + if userLogin == "" { + userLogin = strings.TrimSpace(login) + } + if userLogin != "" { + user.ID = getUIDByEmail(userLogin) + } + } ctx := orm.NewContext() ctx.DirectAccess() - if err := orm.Update(ctx, user); err != nil { + if err := persistPasswordChallengeUpgrade(ctx, user); err != nil { log.Warnf("failed to persist password challenge for user [%s]: %v", user.Email, err) } } diff --git a/modules/security/rbac/account_login_test.go b/modules/security/rbac/account_login_test.go index d2cde6608..ed78ef84f 100644 --- a/modules/security/rbac/account_login_test.go +++ b/modules/security/rbac/account_login_test.go @@ -32,6 +32,8 @@ import ( "testing" "time" + "golang.org/x/crypto/bcrypt" + "infini.sh/framework/core/orm" "infini.sh/framework/core/security" replaysecurity "infini.sh/framework/core/security/replay" ) @@ -316,6 +318,68 @@ func TestNewNativeSessionFallsBackToRequestedLogin(t *testing.T) { } } +func TestUpgradePasswordChallengePersistsLegacyAdminByLogin(t *testing.T) { + originalPersist := persistPasswordChallengeUpgrade + defer func() { + persistPasswordChallengeUpgrade = originalPersist + }() + + var persisted *security.UserAccount + persistPasswordChallengeUpgrade = func(ctx *orm.Context, user *security.UserAccount) error { + copied := *user + persisted = &copied + return nil + } + + user := &security.UserAccount{Name: "admin"} + hash, err := bcrypt.GenerateFromPassword([]byte("StrongPassw0rd!"), bcrypt.DefaultCost) + if err != nil { + t.Fatalf("generate password hash: %v", err) + } + user.Password = string(hash) + + upgradePasswordChallenge(user, "admin", "StrongPassw0rd!") + + if persisted == nil { + t.Fatal("expected legacy admin upgrade to be persisted") + } + if persisted.ID != getUIDByEmail("admin") { + t.Fatalf("expected fallback id %q, got %q", getUIDByEmail("admin"), persisted.ID) + } + if persisted.PasswordSalt == "" || persisted.PasswordVerifier == "" { + t.Fatal("expected challenge credentials to be populated before persisting") + } +} + +func TestUpgradePasswordChallengeSkipsExistingChallengeUser(t *testing.T) { + originalPersist := persistPasswordChallengeUpgrade + defer func() { + persistPasswordChallengeUpgrade = originalPersist + }() + + called := false + persistPasswordChallengeUpgrade = func(ctx *orm.Context, user *security.UserAccount) error { + called = true + return nil + } + + user := &security.UserAccount{Email: "admin@example.org"} + if err := security.SetPassword(user, "StrongPassw0rd!"); err != nil { + t.Fatalf("set password: %v", err) + } + + upgradePasswordChallenge(user, user.Email, "StrongPassw0rd!") + if !security.CanUsePasswordChallenge(user) { + t.Fatal("expected challenge material to be available") + } + called = false + + upgradePasswordChallenge(user, user.Email, "StrongPassw0rd!") + if called { + t.Fatal("did not expect already-upgraded account to be persisted again") + } +} + // The framework login response keeps the console frontend contract while the handler // implementation moves from console into framework-owned routes. func TestDecorateLoginResponseAddsConsoleCompatibilityFields(t *testing.T) { From ab83db6677853c50c7f9fd9fadca2a4bbf79764f Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 2 Jun 2026 18:35:27 +0800 Subject: [PATCH 106/137] improve: login with upgrade --- modules/security/rbac/account_login.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/security/rbac/account_login.go b/modules/security/rbac/account_login.go index 39db70d80..7d24ba994 100644 --- a/modules/security/rbac/account_login.go +++ b/modules/security/rbac/account_login.go @@ -38,10 +38,22 @@ import ( "infini.sh/framework/core/util" ) -var persistPasswordChallengeUpgrade = func(ctx *orm.Context, user *security.UserAccount) error { +var defaultPasswordChallengeUpgradePersister = func(ctx *orm.Context, user *security.UserAccount) error { return orm.Save(ctx, user) } +var persistPasswordChallengeUpgrade = defaultPasswordChallengeUpgradePersister + +// RegisterPasswordChallengeUpgradePersister allows applications to override where +// challenge credentials are persisted after a successful legacy password login. +func RegisterPasswordChallengeUpgradePersister(persister func(ctx *orm.Context, user *security.UserAccount) error) { + if persister == nil { + persistPasswordChallengeUpgrade = defaultPasswordChallengeUpgradePersister + return + } + persistPasswordChallengeUpgrade = persister +} + var ( // Keep the password and challenge paths aligned on one user-facing failure message. errInvalidLoginCredentials = errors.New("invalid login or password") From 5b7aad4bad53f8e3da4fc37a6066e3c8b8ecb885 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 10 Jun 2026 14:15:06 +0800 Subject: [PATCH 107/137] fix: hide task and remove node alert with time range --- modules/elastic/metadata.go | 143 +++++++++++++++++++++++++++--------- 1 file changed, 108 insertions(+), 35 deletions(-) diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index 0e616eb67..0732a82b5 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -850,7 +850,11 @@ func (module *ElasticModule) updateNodeInfo(meta *elastic.ElasticsearchMetadata, if moduleConfig.ORMConfig.Enabled { if meta.Config.Source == elastic.ElasticsearchConfigSourceElasticsearch { //todo check whether store elasticsearch change or not - err = saveNodeMetadata(*nodes, meta.Config.ID) + clusterUUID := meta.Config.ClusterUUID + if meta.ClusterState != nil && meta.ClusterState.ClusterUUID != "" { + clusterUUID = meta.ClusterState.ClusterUUID + } + err = saveNodeMetadata(*nodes, meta.Config.ID, clusterUUID) if err != nil { if rate.GetRateLimiterPerSecond(meta.Config.ID, "save_nodes_metadata_on_error", 1).Allow() { log.Errorf("elasticsearch [%v] failed to save nodes info: %v", meta.Config.Name, err) @@ -935,7 +939,7 @@ func setNodeUnknown(clusterID string) bool { nodeAlreadyUnknown[clusterID] = true return true } -func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) error { +func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID, clusterUUID string) error { esConfig := elastic.GetConfig(clusterID) saveNodeMetadataMutex.Lock() defer func() { @@ -945,28 +949,63 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro } }() - queryDslTpl := `{ - "size": 1000, - "query": { - "bool": { - "must": [ - {"term": { - "metadata.cluster_id": { - "value": "%s" - } - }}, - {"term": { - "metadata.category": { - "value": "elasticsearch" - } - }} - ] - } - } -}` - queryDsl := fmt.Sprintf(queryDslTpl, clusterID) + logicalClusterID := clusterID + if clusterUUID != "" { + logicalClusterID = clusterUUID + } + nodeDocID := func(clusterKey, nodeID string) string { + return util.MD5digest(fmt.Sprintf("%s:%s", clusterKey, nodeID)) + } + + must := []util.MapStr{ + { + "term": util.MapStr{ + "metadata.category": util.MapStr{ + "value": "elasticsearch", + }, + }, + }, + } + boolQuery := util.MapStr{ + "must": must, + } + if clusterUUID != "" { + boolQuery["should"] = []util.MapStr{ + { + "term": util.MapStr{ + "metadata.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }, + { + "term": util.MapStr{ + "metadata.labels.cluster_uuid": util.MapStr{ + "value": clusterUUID, + }, + }, + }, + } + boolQuery["minimum_should_match"] = 1 + } else { + must = append(must, util.MapStr{ + "term": util.MapStr{ + "metadata.cluster_id": util.MapStr{ + "value": clusterID, + }, + }, + }) + boolQuery["must"] = must + } + + queryDsl := util.MustToJSONBytes(util.MapStr{ + "size": 1000, + "query": util.MapStr{ + "bool": boolQuery, + }, + }) q := &orm.Query{} - q.RawQuery = []byte(queryDsl) + q.RawQuery = queryDsl err, result := orm.Search(&elastic.NodeConfig{}, q) if err != nil { return err @@ -981,7 +1020,19 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro //nodeMetadatas[nodeID] = nodeInfo if nid, ok := nodeID.(string); ok { if id, ok := nodeInfo["id"]; ok { - nodeIDMap[nid] = id + existingID, hasExisting := nodeIDMap[nid] + canonicalID := nodeDocID(logicalClusterID, nid) + if !hasExisting { + nodeIDMap[nid] = id + } else { + existingIDStr, existingOK := existingID.(string) + idStr, currentOK := id.(string) + if currentOK && idStr == canonicalID { + nodeIDMap[nid] = id + } else if !(existingOK && existingIDStr == canonicalID) { + nodeIDMap[nid] = id + } + } } historyNodeMetadata[nid] = nodeInfo if _, ok = nodes[nid]; !ok { @@ -996,15 +1047,26 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro rawBytes := util.MustToJSONBytes(nodeInfo) currentNodeInfo := util.MapStr{} util.MustFromJSONBytes(rawBytes, ¤tNodeInfo) + canonicalID := nodeDocID(logicalClusterID, rawNodeID) + legacyID := nodeDocID(clusterID, rawNodeID) var innerID interface{} var typ string var changeLog diff.Changelog if rowID, ok := nodeIDMap[rawNodeID]; !ok { //new - newID := fmt.Sprintf("%s:%s", clusterID, rawNodeID) - newID = util.MD5digest(newID) + newID := canonicalID typ = "create" innerID = newID + labels := util.MapStr{ + "transport_address": nodeInfo.TransportAddress, + "ip": nodeInfo.Ip, + "version": nodeInfo.Version, + "roles": nodeInfo.Roles, + "status": "available", + } + if clusterUUID != "" { + labels["cluster_uuid"] = clusterUUID + } nodeMetadata := &elastic.NodeConfig{ Metadata: elastic.NodeMetadata{ ClusterID: clusterID, @@ -1013,13 +1075,7 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro ClusterName: esConfig.Name, NodeName: nodeInfo.Name, Host: nodeInfo.Host, - Labels: util.MapStr{ - "transport_address": nodeInfo.TransportAddress, - "ip": nodeInfo.Ip, - "version": nodeInfo.Version, - "roles": nodeInfo.Roles, - "status": "available", - }, + Labels: labels, }, ID: newID, Timestamp: time.Now(), @@ -1031,7 +1087,7 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro log.Error(err) } } else { - innerID = rowID + innerID = canonicalID typ = "update" if rid, ok := rowID.(string); ok { if historyM, ok := historyNodeMetadata[rawNodeID]; ok { @@ -1052,6 +1108,9 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro "roles": nodeInfo.Roles, "status": "available", } + if clusterUUID != "" { + newLabels["cluster_uuid"] = clusterUUID + } if labels, err := historyM.GetValue("metadata.labels"); err == nil { if labelsM, ok := labels.(map[string]interface{}); ok { if st, ok := labelsM["status"].(string); ok && st == "unavailable" || st == "N/A" { @@ -1096,7 +1155,7 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro Labels: newLabels, Category: "elasticsearch", }, - ID: rid, + ID: canonicalID, Timestamp: time.Now(), Payload: elastic.NodePayload{NodeInfo: &nodeInfo}, } @@ -1105,6 +1164,14 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro if err != nil { log.Error(err) } + if clusterUUID != "" && legacyID != canonicalID && rid == legacyID { + delCtx := orm.NewContext().DirectAccess() + delCtx.Set(orm.CheckExistsBeforeDelete, false) + err = orm.Delete(delCtx, &elastic.NodeConfig{ID: legacyID}) + if err != nil { + log.Error(err) + } + } } } @@ -1153,6 +1220,12 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID string) erro if oldStatus, ok := oldConfig.Metadata.Labels["status"].(string); ok && oldStatus == "unavailable" { continue } + if oldConfig.Metadata.Labels == nil { + oldConfig.Metadata.Labels = util.MapStr{} + } + if clusterUUID != "" { + oldConfig.Metadata.Labels["cluster_uuid"] = clusterUUID + } oldConfig.Metadata.Labels["status"] = "unavailable" oldConfig.Timestamp = time.Now() From 6fae4950d7d2faeea70670bf1e95f0a698734826 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 11 Jun 2026 09:44:38 +0800 Subject: [PATCH 108/137] fix: common alert with the health activity change --- modules/elastic/metadata.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index 0732a82b5..c6202ba1c 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -194,6 +194,12 @@ func updateClusterHealthStatus(clusterID string, healthStatus string) { }, }, } + log.Infof( + "[health_activity_trace] source=updateClusterHealthStatus event=cluster_health_change cluster_id=%s from=%v to=%v", + clusterID, + oldHealthStatus, + healthStatus, + ) if healthStatus == "red" { targetClient := elastic.GetClient(clusterID) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -1131,6 +1137,11 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID, clusterUUID }, }, } + log.Infof( + "[health_activity_trace] source=saveNodeMetadata event=node_health_change cluster_id=%s node_id=%s to=available", + clusterID, + rawNodeID, + ) ctx1 := orm.NewContext().DirectAccess() err = orm.Save(ctx1, activityInfo) if err != nil { @@ -1252,6 +1263,12 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID, clusterUUID }, }, } + log.Infof( + "[health_activity_trace] source=saveNodeMetadata event=node_health_change cluster_id=%s node_id=%s node_uuid=%s to=unavailable", + clusterID, + oldConfig.Metadata.NodeID, + nodeID, + ) err = orm.Save(ctx1, activityInfo) if err != nil { log.Error(err) From f1ec0c62032a6a5bb133008e07f1e07af43ad9ba Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 11 Jun 2026 14:19:52 +0800 Subject: [PATCH 109/137] fix: index recorded and metadata twice to activity --- modules/elastic/module.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/modules/elastic/module.go b/modules/elastic/module.go index fa283aac5..cb4ad48ea 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -490,12 +490,6 @@ func (module *ElasticModule) Start() error { //update module.updateClusterState(cfg1.ID, true) } - - task.RunWithContext("cluster_health_check", func(ctx context.Context) error { - id := task.MustGetString(ctx, "id") - module.clusterHealthCheck(id, true) - return nil - }, context.WithValue(context.Background(), "id", cfg1.ID)) } } return true From b72cb3818beb7ff8dd6ae2be817900c08cd193cf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 11 Jun 2026 06:28:09 +0000 Subject: [PATCH 110/137] fix: resolve merge conflicts with main branch --- core/config/system.go | 5 - core/pipeline/context.go | 4 - core/security/run_as.go | 4 - core/security/session.go | 3 - core/security/validate.go | 13 -- docs/content.en/docs/release-notes/_index.md | 7 - go.mod | 7 - go.sum | 3 - lib/cache/cache_test.go | 12 -- lib/cache/layeredcache_test.go | 12 -- modules/pipeline/model.go | 11 -- modules/pipeline/pipeline.go | 138 ------------------ modules/pipeline/tasks.go | 9 -- modules/security/native/account_login.go | 2 +- modules/security/native/account_login_test.go | 2 +- modules/security/native/user_test.go | 2 +- 16 files changed, 3 insertions(+), 231 deletions(-) diff --git a/core/config/system.go b/core/config/system.go index c518375b1..64fb8bee9 100755 --- a/core/config/system.go +++ b/core/config/system.go @@ -336,10 +336,6 @@ type AuthenticationConfig struct { OAuth map[string]OAuthConfig `config:"oauth"` } -<<<<<<< HEAD -type AccessTokenConfig struct { - Native RealmConfig `config:"native"` -======= // AccessTokenConfig controls API access-token management. // // When Native is true (default when the native realm is enabled) tokens are @@ -349,7 +345,6 @@ type AccessTokenConfig struct { type AccessTokenConfig struct { Enabled bool `config:"enabled"` Native bool `config:"native"` ->>>>>>> origin/main } type HTTPBasicAuthProvider struct { diff --git a/core/pipeline/context.go b/core/pipeline/context.go index d90ef2cf7..4c8c6e88d 100755 --- a/core/pipeline/context.go +++ b/core/pipeline/context.go @@ -299,7 +299,6 @@ func (ctx *Context) Errors() []error { return ctx.processErrs } -<<<<<<< HEAD func (ctx *Context) GetResultState() RunningState { ctx.stateLock.Lock() defer ctx.stateLock.Unlock() @@ -314,10 +313,7 @@ func (ctx *Context) GetResultError() string { return formatPipelineResultError(ctx.exitErr, ctx.processErrs) } -// Pause will pause the pipeline running loop until Resume called -======= // Pause suspends the goroutine that is running this pipeline. ->>>>>>> origin/main func (ctx *Context) Pause() { ctx.stateLock.Lock() if ctx.isPaused { diff --git a/core/security/run_as.go b/core/security/run_as.go index e5080a463..d573d717a 100644 --- a/core/security/run_as.go +++ b/core/security/run_as.go @@ -8,11 +8,7 @@ import ( "context" ) -<<<<<<< HEAD -func RunAs(ctx context.Context,provider, userID string) context.Context { -======= func RunAs(ctx context.Context, provider, userID string) context.Context { ->>>>>>> origin/main claims := UserSessionInfo{} claims.SetUserID(userID) diff --git a/core/security/session.go b/core/security/session.go index 753709665..f3a21a83d 100644 --- a/core/security/session.go +++ b/core/security/session.go @@ -7,10 +7,7 @@ package security import ( "fmt" "net/http" -<<<<<<< HEAD "sync" -======= ->>>>>>> origin/main "time" "github.com/golang-jwt/jwt/v4" diff --git a/core/security/validate.go b/core/security/validate.go index 7cbc2be55..d6f68a3de 100644 --- a/core/security/validate.go +++ b/core/security/validate.go @@ -79,18 +79,6 @@ func ValidateLogin(w http.ResponseWriter, r *http.Request) (session *UserSession var claims *UserClaims -<<<<<<< HEAD - authHTTPFilterProvider.Range(func(key, value any) bool { - log.Trace("checking auth filter: ", key) - f, ok := value.(HTTPAuthFilterProvider) - if ok { - if claims == nil || !claims.UserSessionInfo.IsValid() { - claims, err = f(w, r) - if claims != nil { - log.Trace("get valid auth info from: ", key) - return false - } -======= authFilterMu.RLock() entries := make([]namedFilterEntry, len(authFilterProviders)) copy(entries, authFilterProviders) @@ -103,7 +91,6 @@ func ValidateLogin(w http.ResponseWriter, r *http.Request) (session *UserSession if claims != nil { log.Debug("get valid auth info from: ", entry.name) break ->>>>>>> origin/main } } } diff --git a/docs/content.en/docs/release-notes/_index.md b/docs/content.en/docs/release-notes/_index.md index 990d0c7e2..4ed7fcf7f 100644 --- a/docs/content.en/docs/release-notes/_index.md +++ b/docs/content.en/docs/release-notes/_index.md @@ -27,14 +27,11 @@ Information about release notes of INFINI Framework is provided here. - feat(client): support token-based authorization #288 - feat: add pluggable sink to host metrics collectors #288 - feat: add access_token to security #359 -<<<<<<< HEAD - feat(security): add native account login challenge, replay protection, and secure transport helpers -======= - feat: smtp processor support parse dynamic content attachments from message #374 - feat: add static rule based authorization #375 - feat: allow to specify OS user when installing the service #380 - feat: only return tokens owned by the current user #381 ->>>>>>> origin/main ### 🐛 Bug fix ### ✈️ Improvements @@ -58,14 +55,10 @@ Information about release notes of INFINI Framework is provided here. - chore: security configuration structure enhanced - chore: remove unused grpc and cuckoo filter" - chore: update seelog for vfs #363 -<<<<<<< HEAD - -======= - chore: udpate update desc to api token, fix permission #372 - chore: unify permission in user's session #379 - chore: skip module start in service control mode #380 - chore: register access_token api only if the feature is enabled #381 ->>>>>>> origin/main ## 1.4.0 (2025-12-19) ### ❌ Breaking changes diff --git a/go.mod b/go.mod index ce4b143db..62958b73a 100644 --- a/go.mod +++ b/go.mod @@ -64,10 +64,6 @@ require ( golang.org/x/text v0.36.0 golang.org/x/time v0.11.0 golang.org/x/tools v0.44.0 -<<<<<<< HEAD - google.golang.org/grpc v1.71.1 -======= ->>>>>>> origin/main gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc gopkg.in/cheggaaa/pb.v1 v1.0.28 gopkg.in/hjson/hjson-go.v3 v3.3.0 @@ -132,10 +128,7 @@ require ( github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect -<<<<<<< HEAD -======= github.com/tetratelabs/wazero v1.9.0 // indirect ->>>>>>> origin/main github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect diff --git a/go.sum b/go.sum index cacd814bc..4b3c93783 100644 --- a/go.sum +++ b/go.sum @@ -330,11 +330,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -<<<<<<< HEAD -======= github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= ->>>>>>> origin/main github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= diff --git a/lib/cache/cache_test.go b/lib/cache/cache_test.go index 2f0d9a054..710e2b02d 100644 --- a/lib/cache/cache_test.go +++ b/lib/cache/cache_test.go @@ -242,28 +242,16 @@ func Test_Cache_ResizeOnTheFly(t *testing.T) { } cache.SetMaxSize(3) time.Sleep(time.Millisecond * 100) -<<<<<<< HEAD - assert.Equal(t, cache.GetDropped(), 2) -======= assert.GreaterOrEqual(t, cache.GetDropped(), 1) ->>>>>>> origin/main assert.Nil(t, cache.Get("0")) assert.NotNil(t, cache.Get("4")) cache.Set("5", 5, time.Minute) time.Sleep(time.Millisecond * 100) -<<<<<<< HEAD - assert.Equal(t, cache.GetDropped(), 1) - assert.Nil(t, cache.Get("2")) - assert.Equal(t, cache.Get("3").Value(), 3) - assert.Equal(t, cache.Get("4").Value(), 4) - assert.Equal(t, cache.Get("5").Value(), 5) -======= assert.GreaterOrEqual(t, cache.GetDropped(), 1) item5 := cache.Get("5") assert.NotNil(t, item5) assert.Equal(t, item5.Value(), 5) ->>>>>>> origin/main cache.SetMaxSize(10) cache.Set("6", 6, time.Minute) diff --git a/lib/cache/layeredcache_test.go b/lib/cache/layeredcache_test.go index 268ad4aa2..20a01fe67 100644 --- a/lib/cache/layeredcache_test.go +++ b/lib/cache/layeredcache_test.go @@ -230,28 +230,16 @@ func Test_LayeredCache_ResizeOnTheFly(t *testing.T) { } cache.SetMaxSize(3) time.Sleep(time.Millisecond * 100) -<<<<<<< HEAD - assert.Equal(t, cache.GetDropped(), 2) -======= assert.GreaterOrEqual(t, cache.GetDropped(), 1) ->>>>>>> origin/main assert.Nil(t, cache.Get("0", "a")) assert.NotNil(t, cache.Get("4", "a")) cache.Set("5", "a", 5, time.Minute) time.Sleep(time.Millisecond * 100) -<<<<<<< HEAD - assert.Equal(t, cache.GetDropped(), 1) - assert.Nil(t, cache.Get("2", "a")) - assert.Equal(t, cache.Get("3", "a").Value(), 3) - assert.Equal(t, cache.Get("4", "a").Value(), 4) - assert.Equal(t, cache.Get("5", "a").Value(), 5) -======= assert.GreaterOrEqual(t, cache.GetDropped(), 1) item5 := cache.Get("5", "a") assert.NotNil(t, item5) assert.Equal(t, item5.Value(), 5) ->>>>>>> origin/main cache.SetMaxSize(10) cache.Set("6", "a", 6, time.Minute) diff --git a/modules/pipeline/model.go b/modules/pipeline/model.go index b2e3d7983..0fce2368b 100644 --- a/modules/pipeline/model.go +++ b/modules/pipeline/model.go @@ -30,7 +30,6 @@ import ( "infini.sh/framework/core/util" ) -<<<<<<< HEAD type PipelineStatus struct { State pipeline.RunningState `json:"state"` LastRunState pipeline.RunningState `json:"last_run_state,omitempty"` @@ -48,14 +47,4 @@ type PipelineTaskStatus = PipelineStatus type PipelineResult struct { Success bool `json:"success"` Error string `json:"error,omitempty"` -======= -type PipelineTaskStatus struct { - State pipeline.RunningState `json:"state"` - CreateTime time.Time `json:"create_time"` - StartTime *time.Time `json:"start_time"` - EndTime *time.Time `json:"end_time"` - Context util.MapStr `json:"context"` - Config *pipeline.PipelineConfigV2 `json:"config"` - Processors []map[string]interface{} `json:"processor"` ->>>>>>> origin/main } diff --git a/modules/pipeline/pipeline.go b/modules/pipeline/pipeline.go index 45c9b72f3..7bedfeccc 100644 --- a/modules/pipeline/pipeline.go +++ b/modules/pipeline/pipeline.go @@ -14,147 +14,9 @@ import ( "infini.sh/framework/core/pipeline" ) -<<<<<<< HEAD -type PipeModule struct { - api.Handler - closed atomic.Bool - - pipelines sync.Map - configs sync.Map - contexts sync.Map -} - -func (module *PipeModule) Name() string { - return "pipeline" -} - -var moduleCfg = struct { - PipelineEnabledByDefault bool `config:"pipeline_enabled_by_default"` -}{PipelineEnabledByDefault: true} - -func (module *PipeModule) Setup() { - if global.Env().IsDebug { - log.Debug("pipeline framework config: ", moduleCfg) - } - - ok, err := env.ParseConfig("preference", &moduleCfg) - if ok && err != nil && global.Env().SystemConfig.Configs.PanicOnConfigError { - panic(err) - } - - module.pipelines = sync.Map{} - module.contexts = sync.Map{} - module.configs = sync.Map{} - - pipeline.RegisterProcessorPlugin("dag", pipeline.NewDAGProcessor) - pipeline.RegisterProcessorPlugin("echo", NewEchoProcessor) - - api.HandleAPIMethod(api.GET, "/pipeline/tasks/", module.getPipelinesHandler) - api.HandleAPIMethod(api.POST, "/pipeline/tasks/_search", module.searchPipelinesHandler) - api.HandleAPIMethod(api.POST, "/pipeline/tasks/", module.createPipelineHandler) - api.HandleAPIMethod(api.GET, "/pipeline/task/:id", module.getPipelineHandler) - api.HandleAPIMethod(api.DELETE, "/pipeline/task/:id", module.deletePipelineHandler) - api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_start", module.startTaskHandler) - api.HandleAPIMethod(api.POST, "/pipeline/task/:id/_stop", module.stopTaskHandler) - -} - -func (module *PipeModule) startTask(taskID string) (exists bool) { - if module.closed.Load() { - return false - } - - ctx, ok := module.contexts.Load(taskID) - if !ok { - return - } - v1, ok := ctx.(*pipeline.Context) - if !ok { - return - } - - exists = true - - // Mark exited pipeline to start again - if v1.IsExit() { - v1.Restart() - } - // Resume pipeline loop - if v1.IsPause() { - // Mark pipeline status as starting - v1.Starting() - v1.Resume() - } - - return -} - -// stopTask will cancel the current pipeline context, abort the pipeline execution. -func (module *PipeModule) stopTask(taskID string) (exists bool) { - ctx, ok := module.contexts.Load(taskID) - if !ok { - return - } - v1, ok := ctx.(*pipeline.Context) - if !ok { - return - } - - exists = true - - if global.Env().IsDebug { - if rate.GetRateLimiterPerSecond("pipeline", "shutdown "+taskID+string(v1.GetRunningState()), 1).Allow() { - log.Trace("start shutting down pipeline:", taskID, ",state:", v1.GetRunningState()) - } - } - - // Mark pipeline as exited - v1.Exit() - // Mark pipeline as STOPPING as needed - v1.Stopping() - // call cancelFunc(), will mark IsCanceled asynchronously - v1.CancelTask() - - return -} - -// deleteTask will clean all in-memory states and release the pipeline context -func (module *PipeModule) deleteTask(taskID string) { - if ctx, ok := module.contexts.Load(taskID); ok { - if v1, ok := ctx.(*pipeline.Context); ok && !v1.IsLoopReleased() { - module.stopAndWaitForRelease([]string{taskID}, time.Minute) - } - } - module.pipelines.Delete(taskID) - module.configs.Delete(taskID) - module.releaseContext(taskID) - module.contexts.Delete(taskID) -} - -// releaseContext will release the task context -func (module *PipeModule) releaseContext(taskID string) { - ctx, ok := module.contexts.Load(taskID) - if ok { - v1, ok := ctx.(*pipeline.Context) - if ok { - pipeline.ReleaseContext(v1) - if v1.IsPause() { - // release loop - v1.Resume() - } - } - } -} - -func getPipelineConfig() ([]pipeline.PipelineConfigV2, error) { - configFile := global.Env().GetConfigFile() - configDir := global.Env().GetConfigDir() - parentCfg, err := config.LoadFile(configFile) -======= func (h *PipeModule) createPipelineHandler(w http.ResponseWriter, req *http.Request, ps httprouter.Params) { var obj = &pipeline.PipelineConfigV2{} err := h.DecodeJSON(req, obj) ->>>>>>> origin/main if err != nil { h.WriteError(w, err.Error(), http.StatusBadRequest) return diff --git a/modules/pipeline/tasks.go b/modules/pipeline/tasks.go index 80e6d1805..b516de9e1 100644 --- a/modules/pipeline/tasks.go +++ b/modules/pipeline/tasks.go @@ -79,7 +79,6 @@ func (module *PipeModule) getPipelineTaskStatus(id string, config string, proces if !ok { return nil } -<<<<<<< HEAD:modules/pipeline/api.go ret := &PipelineStatus{ State: c1.GetRunningState(), LastRunState: c1.GetResultState(), @@ -93,14 +92,6 @@ func (module *PipeModule) getPipelineTaskStatus(id string, config string, proces Success: c1.GetResultError() == "", Error: c1.GetResultError(), } -======= - ret := &PipelineTaskStatus{ - State: c1.GetRunningState(), - CreateTime: c1.GetCreateTime(), - StartTime: c1.GetStartTime(), - EndTime: c1.GetEndTime(), - Context: c1.CloneData(), ->>>>>>> origin/main:modules/pipeline/tasks.go } if config != "false" { v1, ok := module.configs.Load(id) diff --git a/modules/security/native/account_login.go b/modules/security/native/account_login.go index 7d24ba994..6eec79790 100644 --- a/modules/security/native/account_login.go +++ b/modules/security/native/account_login.go @@ -21,7 +21,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package rbac +package native import ( "errors" diff --git a/modules/security/native/account_login_test.go b/modules/security/native/account_login_test.go index ed78ef84f..9f64ec30a 100644 --- a/modules/security/native/account_login_test.go +++ b/modules/security/native/account_login_test.go @@ -21,7 +21,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package rbac +package native import ( "bytes" diff --git a/modules/security/native/user_test.go b/modules/security/native/user_test.go index 163f138eb..f1e8fc82e 100644 --- a/modules/security/native/user_test.go +++ b/modules/security/native/user_test.go @@ -21,7 +21,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -package rbac +package native import ( "strings" From 121bf920722a22446c46f9e67e97a27ab2e38ff5 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 11 Jun 2026 14:28:15 +0800 Subject: [PATCH 111/137] fix: mTLS skip domain verify --- core/api/client.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/api/client.go b/core/api/client.go index 56a84225a..c16bf177d 100755 --- a/core/api/client.go +++ b/core/api/client.go @@ -137,11 +137,10 @@ func GetClientTLSConfig(tlsConfig *config.TLSConfig) (*tls.Config, error) { clientConfig.ServerName = "localhost" } - //skip domain verify if skip tls verify - if !tlsConfig.TLSInsecureSkipVerify { - if tlsConfig.SkipDomainVerify { - clientConfig.VerifyPeerCertificate = util.GetSkipHostnameVerifyFunc(pool) - } + // Skip hostname verification while still validating the certificate chain. + if tlsConfig.SkipDomainVerify && !tlsConfig.TLSInsecureSkipVerify { + clientConfig.InsecureSkipVerify = true + clientConfig.VerifyPeerCertificate = util.GetSkipHostnameVerifyFunc(pool) } return clientConfig, nil From 1164ae08920abe23613bf8751b3cd5fb1157fd11 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 11 Jun 2026 14:33:41 +0800 Subject: [PATCH 112/137] chore: add mTLS skip domain verify test --- core/api/client_test.go | 76 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 core/api/client_test.go diff --git a/core/api/client_test.go b/core/api/client_test.go new file mode 100644 index 000000000..2f965e349 --- /dev/null +++ b/core/api/client_test.go @@ -0,0 +1,76 @@ +package api + +import ( + "crypto/tls" + "net" + "os" + "path/filepath" + "testing" + + "infini.sh/framework/core/config" + "infini.sh/framework/core/util" +) + +func TestGetClientTLSConfigSkipDomainVerifyAllowsHostnameMismatch(t *testing.T) { + rootCert, rootKey, rootCertPEM := util.GetRootCert() + serverCertPEM, serverKeyPEM, err := util.GenerateServerCert(rootCert, rootKey, rootCertPEM, nil) + if err != nil { + t.Fatalf("generate server cert: %v", err) + } + + dir := t.TempDir() + caFile := filepath.Join(dir, "ca.crt") + serverCertFile := filepath.Join(dir, "server.crt") + serverKeyFile := filepath.Join(dir, "server.key") + + if err := os.WriteFile(caFile, rootCertPEM, 0600); err != nil { + t.Fatalf("write ca cert: %v", err) + } + if err := os.WriteFile(serverCertFile, serverCertPEM, 0600); err != nil { + t.Fatalf("write server cert: %v", err) + } + if err := os.WriteFile(serverKeyFile, serverKeyPEM, 0600); err != nil { + t.Fatalf("write server key: %v", err) + } + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("create listener: %v", err) + } + defer ln.Close() + + serverCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile) + if err != nil { + t.Fatalf("load server cert: %v", err) + } + ln = tls.NewListener(ln, &tls.Config{Certificates: []tls.Certificate{serverCert}}) + + done := make(chan struct{}) + go func() { + defer close(done) + conn, err := ln.Accept() + if err != nil { + return + } + if tlsConn, ok := conn.(*tls.Conn); ok { + _ = tlsConn.Handshake() + } + _ = conn.Close() + }() + + cfg, err := GetClientTLSConfig(&config.TLSConfig{ + TLSCACertFile: caFile, + SkipDomainVerify: true, + TLSInsecureSkipVerify: false, + }) + if err != nil { + t.Fatalf("get client tls config: %v", err) + } + + conn, err := tls.Dial("tcp", ln.Addr().String(), cfg) + if err != nil { + t.Fatalf("tls dial: %v", err) + } + _ = conn.Close() + <-done +} From 85bf5aeebe4badf75c5a78fa3f7f677cc19f9033 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 11 Jun 2026 16:15:31 +0800 Subject: [PATCH 113/137] fix: correct GetPipelinesResponse and GetPipelineTasksResponse type alias GetPipelineTasksResponse should be an alias of GetPipelinesResponse, not the other way around. The primary type GetPipelinesResponse is the concrete map type. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- modules/pipeline/proto.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pipeline/proto.go b/modules/pipeline/proto.go index cde467703..7bc29ca39 100644 --- a/modules/pipeline/proto.go +++ b/modules/pipeline/proto.go @@ -25,7 +25,7 @@ package pipeline import "infini.sh/framework/core/pipeline" -type GetPipelineTasksResponse map[string]*PipelineTaskStatus +type GetPipelinesResponse map[string]*PipelineStatus type GetPipelineTasksResponse = GetPipelinesResponse From 2fb02a59a5662cb10782d889261d5e9eb7db7243 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:26:07 +0000 Subject: [PATCH 114/137] fix: remove references to deleted Permissions field in UserSessionInfo --- modules/security/account/refresh.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/security/account/refresh.go b/modules/security/account/refresh.go index 14b0e92d4..5d1f51fa7 100644 --- a/modules/security/account/refresh.go +++ b/modules/security/account/refresh.go @@ -75,13 +75,13 @@ func buildRefreshedSession(reqUser *security.UserSessionInfo) (*security.UserSes } sessionUser = &security.UserSessionInfo{ - Provider: provider, - Login: login, - Roles: append([]string(nil), account.Roles...), - Permissions: append([]security.PermissionKey(nil), reqUser.Permissions...), - LastLogin: reqUser.LastLogin, + Provider: provider, + Login: login, + Roles: append([]string(nil), account.Roles...), + LastLogin: reqUser.LastLogin, } sessionUser.SetUserID(account.ID) + sessionUser.UserAssignedPermission = security.NewUserAssignedPermission(security.GetAllPermissionsForUser(sessionUser), nil) return sessionUser, nil } @@ -91,12 +91,12 @@ func cloneSessionUser(reqUser *security.UserSessionInfo) *security.UserSessionIn } sessionUser := &security.UserSessionInfo{ - Provider: reqUser.Provider, - Login: reqUser.Login, - Roles: append([]string(nil), reqUser.Roles...), - Permissions: append([]security.PermissionKey(nil), reqUser.Permissions...), - LastLogin: reqUser.LastLogin, + Provider: reqUser.Provider, + Login: reqUser.Login, + Roles: append([]string(nil), reqUser.Roles...), + LastLogin: reqUser.LastLogin, } sessionUser.SetUserID(reqUser.UserID) + sessionUser.UserAssignedPermission = security.NewUserAssignedPermission(security.GetAllPermissionsForUser(sessionUser), nil) return sessionUser } From b222e26d0f1136e879e0267fe8ac0e67e433ceab Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 11 Jun 2026 09:29:04 +0000 Subject: [PATCH 115/137] fix: remove references to deleted Permissions field in UserSessionInfo --- config/generated.go | 10 ++++----- core/security/role_registry.go | 8 ++++---- core/security/user_session.go | 2 +- go.mod | 1 - go.sum | 37 ---------------------------------- 5 files changed, 10 insertions(+), 48 deletions(-) diff --git a/config/generated.go b/config/generated.go index 7fe3857a4..2b3afa066 100644 --- a/config/generated.go +++ b/config/generated.go @@ -1,11 +1,11 @@ package config -const LastCommitLog = "4c9e3f77d45b5b5b12af78e29b2e2f13096d319b" +const LastCommitLog = "2fb02a59a5662cb10782d889261d5e9eb7db7243" -const BuildDate = "2026-05-21T08:45:43Z" +const BuildDate = "2026-06-11T09:27:13Z" -const EOLDate = "2023-12-31T10:10:10Z" +const EOLDate = "2023-12-31T10:10:10Z" -const Version = "1.0.0_SNAPSHOT" +const Version = "1.0.0_SNAPSHOT" -const BuildNumber = "001" +const BuildNumber = "001" diff --git a/core/security/role_registry.go b/core/security/role_registry.go index 18ec757db..f9b0fb724 100644 --- a/core/security/role_registry.go +++ b/core/security/role_registry.go @@ -160,8 +160,8 @@ func (rr *RoleRegistry) GetPermissionsForRole(role string) ([]PermissionKey, boo } func GetAllPermissionsForUser(user *UserSessionInfo) []PermissionKey { - if user==nil{ - return []PermissionKey{} + if user == nil { + return []PermissionKey{} } permissions := user.GetPermissionKeys() @@ -194,8 +194,8 @@ func GetAllPermissionsForUser(user *UserSessionInfo) []PermissionKey { } func getPermissionKeysByUser(user *UserSessionInfo) ([]PermissionKey, error) { - if user==nil{ - return []PermissionKey{},nil + if user == nil { + return []PermissionKey{}, nil } ctx1 := context.Background() diff --git a/core/security/user_session.go b/core/security/user_session.go index 3ed4d123a..faef11760 100644 --- a/core/security/user_session.go +++ b/core/security/user_session.go @@ -56,7 +56,7 @@ type UserSessionInfo struct { Login string `json:"login"` //auth login //system level security's info - Roles []string `json:"roles"` + Roles []string `json:"roles"` //private fields UserID string `json:"userid"` //system level user ID diff --git a/go.mod b/go.mod index 62958b73a..67a648fac 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,6 @@ require ( gopkg.in/hjson/hjson-go.v3 v3.3.0 gopkg.in/square/go-jose.v2 v2.6.0 gopkg.in/yaml.v2 v2.4.0 - infini.sh/license v0.0.0-00010101000000-000000000000 k8s.io/api v0.32.3 k8s.io/apimachinery v0.32.3 ) diff --git a/go.sum b/go.sum index 4b3c93783..9d0d51a96 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= code.pfad.fr/check v1.1.0 h1:GWvjdzhSEgHvEHe2uJujDcpmZoySKuHQNrZMfzfO0bE= -code.pfad.fr/check v1.1.0/go.mod h1:NiUH13DtYsb7xp5wll0U4SXx7KhXQVCtRgdC96IPfoM= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/Azure/go-ntlmssp v0.1.0 h1:DjFo6YtWzNqNvQdrwEyr/e4nhU3vRiwenz5QX7sFz+A= github.com/Azure/go-ntlmssp v0.1.0/go.mod h1:NYqdhxd/8aAct/s4qSYZEerdPuH1liG2/X9DiVTbhpk= @@ -13,7 +12,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI= -github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/arl/statsviz v0.6.0 h1:jbW1QJkEYQkufd//4NDYRSNBpwJNrdzPahF7ZmoGdyE= @@ -42,7 +40,6 @@ github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh github.com/dgraph-io/ristretto/v2 v2.2.0 h1:bkY3XzJcXoMuELV8F+vS8kzNgicwQFAaGINAEJdWGOM= github.com/dgraph-io/ristretto/v2 v2.2.0/go.mod h1:RZrm63UmcBAaYWC1DotLYBmTvgkrs0+XhBd7Npn7/zI= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= -github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -58,7 +55,6 @@ github.com/evanphx/json-patch v0.0.0-20200808040245-162e5629780b/go.mod h1:NAJj0 github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= @@ -72,8 +68,6 @@ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-ldap/ldap/v3 v3.2.4/go.mod h1:iYS1MdmrmceOJ1QOTnRXrIs7i3kloqtmGQjRvjKpyMg= github.com/go-ldap/ldap/v3 v3.4.13 h1:+x1nG9h+MZN7h/lUi5Q3UZ0fJ1GyDQYbPvbuH38baDQ= github.com/go-ldap/ldap/v3 v3.4.13/go.mod h1:LxsGZV6vbaK0sIvYfsv47rfh4ca0JXokCoKjZxsszv0= @@ -150,26 +144,13 @@ github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2e github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20170920190843-316c5e0ff04e/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= -github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/log15 v0.0.0-20170622235902-74a0988b5f80/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= -github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= -github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= -github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= -github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= -github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= -github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= -github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= -github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= -github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= -github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= -github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e h1:ZZCvgaRDZg1gC9/1xrsgaJzQUCQgniKtw0xjWywWAOE= github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e/go.mod h1:+rHyWac2R9oAZwFe1wGY2HBzFJJy++RHBg1cU23NkD8= @@ -193,16 +174,9 @@ github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzh github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/letsencrypt/challtestsrv v1.4.2 h1:0ON3ldMhZyWlfVNYYpFuWRTmZNnyfiL9Hh5YzC3JVwU= -github.com/letsencrypt/challtestsrv v1.4.2/go.mod h1:GhqMqcSoeGpYd5zX5TgwA6er/1MbWzx/o7yuuVya+Wk= github.com/letsencrypt/pebble/v2 v2.10.0 h1:Wq6gYXlsY6ubqI3hhxsTzdyotvfdjFBxuwYqCLCnj/U= -github.com/letsencrypt/pebble/v2 v2.10.0/go.mod h1:Sk8cmUIPcIdv2nINo+9PB4L+ZBhzY+F9A1a/h/xmWiQ= github.com/libdns/libdns v1.1.1 h1:wPrHrXILoSHKWJKGd0EiAVmiJbFShguILTg9leS/P/U= github.com/libdns/libdns v1.1.1/go.mod h1:4Bj9+5CQiNMVGf87wjX4CY3HQJypUHRuLvlsfsZqLWQ= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -219,7 +193,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.2/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mholt/acmez/v3 v3.1.6 h1:eGVQNObP0pBN4sxqrXeg7MYqTOWyoiYpQqITVWlrevk= @@ -252,17 +225,13 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/nsqio/nsq v1.3.0 h1:v7NtyO844ieTIOCQEqQ7IUSSi1ImhgrTTto1rgIYGEU= github.com/nsqio/nsq v1.3.0/go.mod h1:RxNr6UC0kSkNF44LnJrlN3U3CQnQGTXk+QKfSZLzqvc= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= @@ -281,7 +250,6 @@ github.com/r3labs/diff/v2 v2.15.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= @@ -357,11 +325,9 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= -github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= -github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/zeebo/sbloom v0.0.0-20151106181526-405c65bd9be0 h1:EAluI/s9FYrMnDGmyXB6eKkjSNyn7lmSdvX975YHZnY= github.com/zeebo/sbloom v0.0.0-20151106181526-405c65bd9be0/go.mod h1:J0OA/x7vNUsWZ88/oJ0BPtebbGfjvSW1lA07GinZNLM= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= @@ -373,7 +339,6 @@ go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJ go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= @@ -459,7 +424,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -470,7 +434,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 615ca023016f616b462b26e66259aa1be7bce6c1 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 11 Jun 2026 17:36:26 +0800 Subject: [PATCH 116/137] chore: revert the generated info --- config/generated.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/generated.go b/config/generated.go index 2b3afa066..baf497913 100644 --- a/config/generated.go +++ b/config/generated.go @@ -1,11 +1,11 @@ package config -const LastCommitLog = "2fb02a59a5662cb10782d889261d5e9eb7db7243" +const LastCommitLog = "N/A" -const BuildDate = "2026-06-11T09:27:13Z" +const BuildDate = "N/A" -const EOLDate = "2023-12-31T10:10:10Z" +const EOLDate = "N/A" -const Version = "1.0.0_SNAPSHOT" +const Version = "0.0.1-SNAPSHOT" -const BuildNumber = "001" +const BuildNumber = "001" From 1a64d24bc9df13cb5fa587ee6b9affb4091ec7de Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 11 Jun 2026 18:32:21 +0800 Subject: [PATCH 117/137] chore: revert app --- app.go | 29 +++++++---------------------- app_test.go | 20 -------------------- 2 files changed, 7 insertions(+), 42 deletions(-) delete mode 100644 app_test.go diff --git a/app.go b/app.go index 3d3e5a60e..f2f3c6142 100755 --- a/app.go +++ b/app.go @@ -33,7 +33,6 @@ import ( "fmt" "os" "os/signal" - "path/filepath" "runtime" "runtime/debug" "sync" @@ -87,18 +86,6 @@ type App struct { svcUser string } -func getServiceWorkingDirectory() string { - executablePath, err := os.Executable() - if err == nil { - return filepath.Dir(executablePath) - } - workdir, err := os.Getwd() - if err != nil { - panic(err) - } - return workdir -} - const ( env_SILENT_GREETINGS = "SILENT_GREETINGS" env_SERVICE_NAME = "SERVICE_NAME" @@ -217,12 +204,7 @@ func (app *App) initWithFlags() { app.configFile = app.environment.GetAppLowercaseName() + ".yml" } - resolvedConfigFile, err := util.GetFileAbsPath(app.configFile, app.environment.IgnoreOnConfigMissing) - if err != nil { - log.Errorf("failed to locate main config file [%v]: %v", app.configFile, err) - os.Exit(1) - } - app.configFile = resolvedConfigFile + app.configFile = util.TryGetFileAbsPath(app.configFile, app.environment.IgnoreOnConfigMissing) if !util.FileExists(app.configFile) { fmt.Println(errors.Errorf("main config file [%v] not exists", app.configFile)) @@ -233,7 +215,7 @@ func (app *App) initWithFlags() { app.environment.SetConfigFile(app.configFile) - err = app.environment.InitPaths(app.configFile) + err := app.environment.InitPaths(app.configFile) if err != nil { panic(err) } @@ -595,7 +577,10 @@ func (app *App) Run() { svcOptions["SuccessExitStatus"] = "1 2 8 SIGKILL" svcOptions["LimitNOFILE"] = 1024000 - workdir := getServiceWorkingDirectory() + workdir, err := os.Getwd() + if err != nil { + panic(err) + } serviceName := app.environment.GetAppLowercaseName() if v, ok := os.LookupEnv(env_SERVICE_NAME); ok { @@ -636,4 +621,4 @@ func (app *App) Run() { if err != nil { log.Error(err) } -} +} \ No newline at end of file diff --git a/app_test.go b/app_test.go deleted file mode 100644 index 702a85f48..000000000 --- a/app_test.go +++ /dev/null @@ -1,20 +0,0 @@ -package framework - -import ( - "os" - "path/filepath" - "testing" -) - -func TestGetServiceWorkingDirectoryUsesExecutableDir(t *testing.T) { - executablePath, err := os.Executable() - if err != nil { - t.Fatalf("failed to get executable path: %v", err) - } - - got := getServiceWorkingDirectory() - want := filepath.Dir(executablePath) - if got != want { - t.Fatalf("expected service working directory %q, got %q", want, got) - } -} From 47c5590fe6ddb53d017fb2fd7728e03a12f7cff1 Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 11 Jun 2026 19:25:30 +0800 Subject: [PATCH 118/137] chore: revert unnecessary change --- app.go | 2 +- core/api/security_guard.go | 26 ------------ core/api/security_guard_test.go | 75 --------------------------------- go.mod | 2 - modules/api/api.go | 5 +-- modules/web/web.go | 3 -- 6 files changed, 3 insertions(+), 110 deletions(-) delete mode 100644 core/api/security_guard.go delete mode 100644 core/api/security_guard_test.go diff --git a/app.go b/app.go index f2f3c6142..a9c7a7e47 100755 --- a/app.go +++ b/app.go @@ -621,4 +621,4 @@ func (app *App) Run() { if err != nil { log.Error(err) } -} \ No newline at end of file +} diff --git a/core/api/security_guard.go b/core/api/security_guard.go deleted file mode 100644 index 7cff00683..000000000 --- a/core/api/security_guard.go +++ /dev/null @@ -1,26 +0,0 @@ -package api - -import ( - "strings" - - "infini.sh/framework/core/config" - "infini.sh/framework/core/errors" -) - -func ValidateServerExposureConfig(cfg *config.SystemConfig) error { - if cfg == nil { - return nil - } - if cfg.APIConfig.Enabled && !cfg.APIConfig.Security.Enabled { - return errors.Errorf("unsafe config: api.enabled requires api.security.enabled") - } - if cfg.WebAppConfig.Enabled && cfg.WebAppConfig.EmbeddingAPI { - return errors.Errorf("unsafe config: web.embedding_api is forbidden; use protected UI routes instead") - } - if cfg.APIConfig.Security.Enabled { - if strings.TrimSpace(cfg.APIConfig.Security.Username) == "" { - return errors.Errorf("unsafe config: api.security.username is required when api.security.enabled is true") - } - } - return nil -} diff --git a/core/api/security_guard_test.go b/core/api/security_guard_test.go deleted file mode 100644 index 610b01361..000000000 --- a/core/api/security_guard_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package api - -import ( - "strings" - "testing" - - "infini.sh/framework/core/config" -) - -func TestValidateServerExposureConfig(t *testing.T) { - tests := []struct { - name string - cfg config.SystemConfig - wantErr string - }{ - { - name: "safe api config", - cfg: config.SystemConfig{ - APIConfig: config.APIConfig{ - Enabled: true, - Security: config.APISecurityConfig{ - Enabled: true, - Username: "api-user", - }, - }, - }, - }, - { - name: "reject insecure api", - cfg: config.SystemConfig{ - APIConfig: config.APIConfig{ - Enabled: true, - }, - }, - wantErr: "api.enabled requires api.security.enabled", - }, - { - name: "reject embedded api on web", - cfg: config.SystemConfig{ - WebAppConfig: config.WebAppConfig{ - Enabled: true, - EmbeddingAPI: true, - }, - }, - wantErr: "web.embedding_api is forbidden", - }, - { - name: "reject missing api username", - cfg: config.SystemConfig{ - APIConfig: config.APIConfig{ - Enabled: true, - Security: config.APISecurityConfig{ - Enabled: true, - }, - }, - }, - wantErr: "api.security.username is required", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := ValidateServerExposureConfig(&tt.cfg) - if tt.wantErr == "" { - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - return - } - if err == nil || !strings.Contains(err.Error(), tt.wantErr) { - t.Fatalf("expected error containing %q, got %v", tt.wantErr, err) - } - }) - } -} diff --git a/go.mod b/go.mod index 67a648fac..917e3a71d 100644 --- a/go.mod +++ b/go.mod @@ -154,5 +154,3 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) - -replace infini.sh/license => ../license diff --git a/modules/api/api.go b/modules/api/api.go index 613fe1626..e88cc4133 100755 --- a/modules/api/api.go +++ b/modules/api/api.go @@ -187,10 +187,9 @@ func (module *APIModule) Setup() { } func (module *APIModule) Start() error { - if err := api.ValidateServerExposureConfig(global.Env().SystemConfig); err != nil { - return err + if global.Env().SystemConfig.APIConfig.Enabled { + api.StartAPI() } - api.StartAPI() return nil } diff --git a/modules/web/web.go b/modules/web/web.go index 4cada04a9..c46979881 100755 --- a/modules/web/web.go +++ b/modules/web/web.go @@ -57,9 +57,6 @@ func (module *WebModule) Setup() { func (module *WebModule) Start() error { if global.Env().SystemConfig.WebAppConfig.Enabled { - if err := uis.ValidateServerExposureConfig(global.Env().SystemConfig); err != nil { - return err - } uis.StartWeb(global.Env().SystemConfig.WebAppConfig) } return nil From e43f1afa936d95dce48e74acb384b7fdb2af68cf Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 11 Jun 2026 19:36:25 +0800 Subject: [PATCH 119/137] chore: revert unnecessary change --- core/env/env.go | 35 +++-------------------------- core/env/env_test.go | 53 +------------------------------------------- 2 files changed, 4 insertions(+), 84 deletions(-) diff --git a/core/env/env.go b/core/env/env.go index 1744e37f3..79f32f960 100755 --- a/core/env/env.go +++ b/core/env/env.go @@ -255,11 +255,7 @@ func (env *Env) InitPaths(cfgPath string) error { if cfgObj, err = config.LoadFile(cfgPath); err != nil { return fmt.Errorf("error loading confiuration file: %v, %w", cfgPath, err) } - if err := cfgObj.Unpack(&env.SystemConfig); err != nil { - return err - } - env.normalizeRelativePaths() - return nil + return cfgObj.Unpack(&env.SystemConfig) } else { if !env.IgnoreOnConfigMissing { return errors.Errorf("config file %v not found", cfgPath) @@ -422,7 +418,6 @@ func (env *Env) loadEnvFromConfigFile(filename string) error { } env.SystemConfig = &tempCfg - env.normalizeRelativePaths() //initialize node config env.findWorkingDir() @@ -486,30 +481,6 @@ func (env *Env) loadEnvFromConfigFile(filename string) error { return nil } -func resolvePathRelativeToExecutable(p string) string { - p = strings.TrimSpace(p) - if p == "" || filepath.IsAbs(p) { - return p - } - - executablePath, err := os.Executable() - if err != nil { - return p - } - return filepath.Join(filepath.Dir(executablePath), p) -} - -func (env *Env) normalizeRelativePaths() { - if env.SystemConfig == nil { - return - } - - env.SystemConfig.PathConfig.Config = resolvePathRelativeToExecutable(env.SystemConfig.PathConfig.Config) - env.SystemConfig.PathConfig.Data = resolvePathRelativeToExecutable(env.SystemConfig.PathConfig.Data) - env.SystemConfig.PathConfig.Log = resolvePathRelativeToExecutable(env.SystemConfig.PathConfig.Log) - env.SystemConfig.PathConfig.Plugin = resolvePathRelativeToExecutable(env.SystemConfig.PathConfig.Plugin) -} - func (env *Env) GetConfigFile() string { return env.configFile } @@ -579,7 +550,7 @@ func ParseConfigSection(cfg *config.Config, configKey string, configInstance int // go-ucfg raises an error if the key does not exist, in which case // we should return and report that the configKey does not exist. if ucfgErr, ok := err.(ucfg.Error); ok && ucfgErr.Reason() == ucfg.ErrMissing { - log.Tracef("config key: %s not found", configKey) + log.Debugf("config key: %s not found", configKey) return false, nil } @@ -874,4 +845,4 @@ func (env *Env) UpdateState(i int32) { func (env *Env) GetState() int32 { return atomic.LoadInt32(&env.state) -} +} \ No newline at end of file diff --git a/core/env/env_test.go b/core/env/env_test.go index b87292741..da9444a4a 100644 --- a/core/env/env_test.go +++ b/core/env/env_test.go @@ -24,8 +24,6 @@ package env import ( - "os" - "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -105,55 +103,6 @@ func TestParseConfigSection_ExistingKey_UnpackFails(t *testing.T) { require.Error(t, err) } -func TestResolvePathRelativeToExecutableUsesExecutableDir(t *testing.T) { - executablePath, err := os.Executable() - require.NoError(t, err) - - got := resolvePathRelativeToExecutable("data") - - assert.Equal(t, filepath.Join(filepath.Dir(executablePath), "data"), got) -} - -func TestNormalizeRelativePathsUsesExecutableDir(t *testing.T) { - executablePath, err := os.Executable() - require.NoError(t, err) - - env := EmptyEnv() - env.SystemConfig.PathConfig.Config = "config" - env.SystemConfig.PathConfig.Data = "data" - env.SystemConfig.PathConfig.Log = "log" - env.SystemConfig.PathConfig.Plugin = "plugin" - - env.normalizeRelativePaths() - - executableDir := filepath.Dir(executablePath) - assert.Equal(t, filepath.Join(executableDir, "config"), env.SystemConfig.PathConfig.Config) - assert.Equal(t, filepath.Join(executableDir, "data"), env.SystemConfig.PathConfig.Data) - assert.Equal(t, filepath.Join(executableDir, "log"), env.SystemConfig.PathConfig.Log) - assert.Equal(t, filepath.Join(executableDir, "plugin"), env.SystemConfig.PathConfig.Plugin) -} - -func TestInitPathsNormalizesRelativePathsFromConfig(t *testing.T) { - executablePath, err := os.Executable() - require.NoError(t, err) - - cfgFile, err := os.CreateTemp("", "env-paths-*.yml") - require.NoError(t, err) - defer os.Remove(cfgFile.Name()) - - _, err = cfgFile.WriteString("path.data: data\npath.log: log\npath.configs: config\n") - require.NoError(t, err) - require.NoError(t, cfgFile.Close()) - - env := EmptyEnv() - require.NoError(t, env.InitPaths(cfgFile.Name())) - - executableDir := filepath.Dir(executablePath) - assert.Equal(t, filepath.Join(executableDir, "data"), env.SystemConfig.PathConfig.Data) - assert.Equal(t, filepath.Join(executableDir, "log"), env.SystemConfig.PathConfig.Log) - assert.Equal(t, filepath.Join(executableDir, "config"), env.SystemConfig.PathConfig.Config) -} - func TestParseConfigSection_KeyExistsButPrimitive_ReturnsError(t *testing.T) { // Key exists but value is primitive (string), not an object. Child returns type error. cfg, err := config.NewConfigFrom(map[string]interface{}{ @@ -166,4 +115,4 @@ func TestParseConfigSection_KeyExistsButPrimitive_ReturnsError(t *testing.T) { assert.False(t, exist) require.Error(t, err) -} +} \ No newline at end of file From 20f3ac0853b737ddfb180da8ae80236c0ba6670c Mon Sep 17 00:00:00 2001 From: medcl Date: Thu, 11 Jun 2026 20:28:02 +0800 Subject: [PATCH 120/137] chore: revert unnecessary change --- core/event/store.go | 71 ++++++-------------------------------- core/event/store_test.go | 74 ---------------------------------------- 2 files changed, 11 insertions(+), 134 deletions(-) delete mode 100644 core/event/store_test.go diff --git a/core/event/store.go b/core/event/store.go index 2e7c34ab9..067893803 100644 --- a/core/event/store.go +++ b/core/event/store.go @@ -36,56 +36,6 @@ import ( "infini.sh/framework/core/util" ) -var pushQueueMessage = queue.Push -var getOrInitQueueConfig = queue.GetOrInitConfig - -func normalizeLabelValue(value interface{}) interface{} { - switch v := value.(type) { - case util.MapStr: - if len(v) == 1 { - if inner, ok := v["value"]; ok { - return normalizeLabelValue(inner) - } - if inner, ok := v["terms"]; ok { - return normalizeLabelValue(inner) - } - } - for key, item := range v { - v[key] = normalizeLabelValue(item) - } - return v - case map[string]interface{}: - if len(v) == 1 { - if inner, ok := v["value"]; ok { - return normalizeLabelValue(inner) - } - if inner, ok := v["terms"]; ok { - return normalizeLabelValue(inner) - } - } - for key, item := range v { - v[key] = normalizeLabelValue(item) - } - return v - case []interface{}: - for i, item := range v { - v[i] = normalizeLabelValue(item) - } - return v - default: - return value - } -} - -func normalizeEventLabels(event *Event) { - if event == nil || event.Metadata.Labels == nil { - return - } - for key, value := range event.Metadata.Labels { - event.Metadata.Labels[key] = normalizeLabelValue(value) - } -} - var meta *AgentMeta func RegisterMeta(m *AgentMeta) { @@ -110,17 +60,15 @@ func SaveWithTimestamp(event *Event, time2 time.Time) error { panic("event can't be nil") } - normalizeEventLabels(event) - if global.Env().IsDebug { - log.Tracef("%v-%v: %v", event.Metadata.Category, event.Metadata.Name, string(util.MustToJSONBytes(event.Metadata))) + log.Debugf("%v-%v: %v", event.Metadata.Category, event.Metadata.Name, string(util.MustToJSONBytes(event.Metadata))) } event.Timestamp = time2 //check event specified queue name if event.QueueName != "" { - return pushQueueMessage(getOrInitQueueConfig(event.QueueName), util.MustToJSONBytes(event)) + return queue.Push(queue.GetOrInitConfig(event.QueueName), util.MustToJSONBytes(event)) } else { //check default queue name if getMeta().DefaultMetricQueueName == "" { @@ -134,7 +82,7 @@ func SaveWithTimestamp(event *Event, time2 time.Time) error { } stats.Increment("metrics.save", event.Metadata.Category, event.Metadata.Name) - return pushQueueMessage(getOrInitQueueConfig(event.QueueName), util.MustToJSONBytes(event)) + return queue.Push(queue.GetOrInitConfig(event.QueueName), util.MustToJSONBytes(event)) } func Save(event *Event) error { @@ -146,8 +94,6 @@ func SaveLog(event *Event) error { panic("event can't be nil") } - normalizeEventLabels(event) - event.Timestamp = time.Now() event.Agent = getMeta() @@ -156,10 +102,15 @@ func SaveLog(event *Event) error { } if global.Env().IsDebug { - log.Tracef("%v-%v: %v, %v", event.Metadata.Category, event.Metadata.Name, util.MustToJSON(event.Metadata), util.MustToJSON(event.Fields)) + log.Debugf("%v-%v: %v, %v", event.Metadata.Category, event.Metadata.Name, util.MustToJSON(event.Metadata), util.MustToJSON(event.Fields)) } stats.Increment("metrics.savelog", event.Metadata.Category, event.Metadata.Name) - return pushQueueMessage(getOrInitQueueConfig(getMeta().LoggingQueueName), util.MustToJSONBytes(event)) -} + err := queue.Push(queue.GetOrInitConfig(getMeta().LoggingQueueName), util.MustToJSONBytes(event)) + if err != nil { + panic(err) + } + + return nil +} \ No newline at end of file diff --git a/core/event/store_test.go b/core/event/store_test.go deleted file mode 100644 index add636993..000000000 --- a/core/event/store_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package event - -import ( - "errors" - "testing" - - "infini.sh/framework/core/queue" - "infini.sh/framework/core/util" -) - -func TestNormalizeEventLabelsFlattensDSLWrappers(t *testing.T) { - item := &Event{ - Metadata: EventMetadata{ - Labels: util.MapStr{ - "cluster_id": util.MapStr{ - "terms": "infini_default_system_cluster", - }, - "cluster_uuid": map[string]interface{}{ - "value": "cluster-uuid", - }, - "roles": []interface{}{ - util.MapStr{"value": "data"}, - map[string]interface{}{"terms": "ingest"}, - }, - }, - }, - } - - normalizeEventLabels(item) - - if got := item.Metadata.Labels["cluster_id"]; got != "infini_default_system_cluster" { - t.Fatalf("expected flattened cluster_id label, got %#v", got) - } - if got := item.Metadata.Labels["cluster_uuid"]; got != "cluster-uuid" { - t.Fatalf("expected flattened cluster_uuid label, got %#v", got) - } - - roles, ok := item.Metadata.Labels["roles"].([]interface{}) - if !ok { - t.Fatalf("expected roles to remain a slice, got %#v", item.Metadata.Labels["roles"]) - } - if len(roles) != 2 || roles[0] != "data" || roles[1] != "ingest" { - t.Fatalf("expected flattened roles entries, got %#v", roles) - } -} - -func TestSaveLogReturnsQueueError(t *testing.T) { - originalPush := pushQueueMessage - originalGetOrInitQueueConfig := getOrInitQueueConfig - originalMeta := meta - t.Cleanup(func() { - pushQueueMessage = originalPush - getOrInitQueueConfig = originalGetOrInitQueueConfig - meta = originalMeta - }) - - pushQueueMessage = func(_ *queue.QueueConfig, _ []byte) error { - return errors.New("readonly") - } - getOrInitQueueConfig = func(_ string) *queue.QueueConfig { - return &queue.QueueConfig{} - } - meta = &AgentMeta{LoggingQueueName: "logging"} - - err := SaveLog(&Event{ - Metadata: EventMetadata{ - Category: "task", - Name: "logging", - }, - }) - if err == nil || err.Error() != "readonly" { - t.Fatalf("expected readonly error, got %v", err) - } -} From 193351d3fb8b927354d0637adbccc39fe6ae3fcd Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 11 Jun 2026 21:52:37 +0800 Subject: [PATCH 121/137] fix: get alias failed with permission --- modules/elastic/adapter/elasticsearch/v0.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/elastic/adapter/elasticsearch/v0.go b/modules/elastic/adapter/elasticsearch/v0.go index 873ede19a..40174c0e0 100755 --- a/modules/elastic/adapter/elasticsearch/v0.go +++ b/modules/elastic/adapter/elasticsearch/v0.go @@ -1531,6 +1531,9 @@ func (c *ESAPIV0) GetAliases() (*map[string]elastic.AliasInfo, error) { resp, err := c.Request(nil, util.Verb_GET, url, nil) if err != nil || resp.StatusCode != 200 { + if err == nil { + return nil, errors.NewWithHTTPCode(resp.StatusCode, string(resp.Body)) + } return nil, err } @@ -1613,6 +1616,9 @@ func (c *ESAPIV0) GetAliasesAndIndices() (*elastic.AliasAndIndicesResponse, erro resp, err := c.Request(nil, util.Verb_GET, url, nil) if err != nil || resp.StatusCode != 200 { + if err == nil { + return nil, errors.NewWithHTTPCode(resp.StatusCode, string(resp.Body)) + } return nil, err } data := map[string]AliasesResponse{} From d41d66858da9ac31e981f9ab06b8c00559555708 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 12 Jun 2026 09:52:26 +0800 Subject: [PATCH 122/137] chore: remove trace log --- core/elastic/domain.go | 3 +++ modules/elastic/metadata.go | 17 ----------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/core/elastic/domain.go b/core/elastic/domain.go index 97bb08ca0..bcac81c46 100644 --- a/core/elastic/domain.go +++ b/core/elastic/domain.go @@ -541,6 +541,9 @@ type ElasticsearchConfig struct { Distribution string `json:"distribution,omitempty" elastic_mapping:"distribution:{type:keyword}"` NoDefaultAuthForAgent bool `json:"no_default_auth_for_agent,omitempty" config:"no_default_auth_for_agent"` MetricCollectionMode string `json:"metric_collection_mode,omitempty" elastic_mapping:"metric_collection_mode:{type:keyword}"` + // AgentCollectionInterval is the default metrics collection interval (in seconds) for all Agent pipelines + // monitoring this cluster. 0 means use the Agent binary default (10 s). Can be overridden per-node via node_settings. + AgentCollectionInterval int `json:"agent_collection_interval,omitempty" elastic_mapping:"agent_collection_interval:{type:integer}"` } const ( diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index c6202ba1c..0732a82b5 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -194,12 +194,6 @@ func updateClusterHealthStatus(clusterID string, healthStatus string) { }, }, } - log.Infof( - "[health_activity_trace] source=updateClusterHealthStatus event=cluster_health_change cluster_id=%s from=%v to=%v", - clusterID, - oldHealthStatus, - healthStatus, - ) if healthStatus == "red" { targetClient := elastic.GetClient(clusterID) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -1137,11 +1131,6 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID, clusterUUID }, }, } - log.Infof( - "[health_activity_trace] source=saveNodeMetadata event=node_health_change cluster_id=%s node_id=%s to=available", - clusterID, - rawNodeID, - ) ctx1 := orm.NewContext().DirectAccess() err = orm.Save(ctx1, activityInfo) if err != nil { @@ -1263,12 +1252,6 @@ func saveNodeMetadata(nodes map[string]elastic.NodesInfo, clusterID, clusterUUID }, }, } - log.Infof( - "[health_activity_trace] source=saveNodeMetadata event=node_health_change cluster_id=%s node_id=%s node_uuid=%s to=unavailable", - clusterID, - oldConfig.Metadata.NodeID, - nodeID, - ) err = orm.Save(ctx1, activityInfo) if err != nil { log.Error(err) From 8042b5eaa75a722b7f2bfd9520300736f3776b03 Mon Sep 17 00:00:00 2001 From: medcl Date: Fri, 12 Jun 2026 14:32:40 +0800 Subject: [PATCH 123/137] chore: remove unused duplicated access_token --- core/access_token/authentication.go | 133 ---------------------------- core/access_token/module.go | 28 ------ 2 files changed, 161 deletions(-) delete mode 100644 core/access_token/authentication.go delete mode 100644 core/access_token/module.go diff --git a/core/access_token/authentication.go b/core/access_token/authentication.go deleted file mode 100644 index a6e9a5236..000000000 --- a/core/access_token/authentication.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (C) INFINI Labs & INFINI LIMITED. -// -// The INFINI Framework is offered under the GNU Affero General Public License v3.0 -// and as commercial software. -// -// For commercial licensing, contact us at: -// - Website: infinilabs.com -// - Email: hello@infini.ltd -// -// Open Source licensed under AGPL V3: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package access_token - -import ( - "net/http" - "strings" - - log "github.com/cihub/seelog" - "github.com/emirpasic/gods/sets/hashset" - "infini.sh/framework/core/credential" - "infini.sh/framework/core/errors" - "infini.sh/framework/core/model" - "infini.sh/framework/core/orm" - "infini.sh/framework/core/util" -) - -type AccessToken struct { - orm.ORMObjectBase - Name string `json:"name,omitempty"` - Description string `json:"description,omitempty"` - Username string `json:"username,omitempty"` - Value string `json:"value,omitempty"` - Permissions []string `json:"permissions,omitempty"` -} - -func (a AccessToken) String() string { - return util.ToJson(a, false) -} - -func ValidatePermissionByAccessToken(req *http.Request) error { - token := strings.TrimSpace(req.Header.Get(model.API_TOKEN)) - if token == "" { - return nil - } - tokenObject, err := GetByToken(token) - if err != nil { - return errors.NewWithHTTPCode(http.StatusUnauthorized, "invalid access token") - } - reqTokenPermissions := req.URL.Query()["permission"] - if len(reqTokenPermissions) == 0 { - return nil - } - userPermissionsSet := hashset.New() - for _, item := range tokenObject.Permissions { - userPermissionsSet.Add(item) - } - for _, permission := range reqTokenPermissions { - if !userPermissionsSet.Contains(permission) { - return errors.NewWithHTTPCode(http.StatusUnauthorized, "invalid access token permissions") - } - } - return nil -} - -func GetByToken(token string) (*AccessToken, error) { - err, result := orm.GetBy("type", credential.AccessToken, credential.Credential{}) - if err != nil { - return nil, err - } - for _, item := range result.Result { - cred := credential.Credential{} - err := util.FromJSONBytes(util.MustToJSONBytes(item), &cred) - if err != nil { - return nil, err - } - payload, err := cred.DecodeAccessToken() - if err != nil { - return nil, err - } - if payload.Value.String() == token { - return &AccessToken{ - ORMObjectBase: cred.ORMObjectBase, - Name: cred.Name, - Description: payload.Description, - Username: payload.Username, - Value: payload.Value.String(), - Permissions: payload.Permissions, - }, nil - } - } - return nil, errors.NewWithHTTPCode(http.StatusNotFound, "access token not found") -} - -func AddPermissionFilterByAccessToken(base []string, req *http.Request) []string { - token := strings.TrimSpace(req.Header.Get(model.API_TOKEN)) - if token == "" { - return base - } - tokenObject, err := GetByToken(token) - if err != nil { - log.Error("error on get access token,", err) - return base - } - if len(tokenObject.Permissions) == 0 { - return base - } - set := hashset.New() - for _, item := range base { - set.Add(item) - } - for _, item := range tokenObject.Permissions { - set.Add(item) - } - values := make([]string, 0, set.Size()) - for _, item := range set.Values() { - if str, ok := item.(string); ok { - values = append(values, str) - } - } - return values -} diff --git a/core/access_token/module.go b/core/access_token/module.go deleted file mode 100644 index 1c74ab78d..000000000 --- a/core/access_token/module.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (C) INFINI Labs & INFINI LIMITED. -// -// The INFINI Framework is offered under the GNU Affero General Public License v3.0 -// and as commercial software. -// -// For commercial licensing, contact us at: -// - Website: infinilabs.com -// - Email: hello@infini.ltd -// -// Open Source licensed under AGPL V3: -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package access_token - -func Init() error { - return nil -} From 0eeeb07c6567f157291567e3848ec69797a998e6 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 12 Jun 2026 15:18:56 +0800 Subject: [PATCH 124/137] improve: metadata sync increasement --- modules/elastic/metadata.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index 0732a82b5..f6bef05c1 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -286,25 +286,23 @@ func (module *ElasticModule) updateClusterState(clusterId string, force bool) { log.Tracef("cluster state updated from version [%v] to [%v]", meta.ClusterState.Version, state.Version) } - oldIndexState, err := kv.GetCompressedValue(elastic.KVElasticIndexMetadata, []byte(clusterId)) - - //TODO locker - if stateChanged || (err == nil && oldIndexState == nil) { - if meta.Config.Source == elastic.ElasticsearchConfigSourceElasticsearch { - if meta.ClusterState == nil || oldIndexState == nil { - //load init state from es when console start - oldIndexState, err = module.loadIndexMetadataFromES(clusterId) - if err != nil { - log.Errorf("failed to load index metadata from es: %v", err) - } - err = kv.AddValueCompressWithTTL(elastic.KVElasticIndexMetadata, []byte(clusterId), oldIndexState, elasticMetadataKVRetention) - if err != nil { + oldIndexState, oldIndexStateErr := kv.GetCompressedValue(elastic.KVElasticIndexMetadata, []byte(clusterId)) + if meta.Config.Source == elastic.ElasticsearchConfigSourceElasticsearch { + if meta.ClusterState == nil || oldIndexState == nil { + // load init state from es when console start + oldIndexState, oldIndexStateErr = module.loadIndexMetadataFromES(clusterId) + if oldIndexStateErr != nil { + log.Errorf("failed to load index metadata from es: %v", oldIndexStateErr) + } else { + if err := kv.AddValueCompressWithTTL(elastic.KVElasticIndexMetadata, []byte(clusterId), oldIndexState, elasticMetadataKVRetention); err != nil { log.Errorf("failed to save index metadata: %v", err) } } - if err == nil { - module.saveIndexMetadata(state, clusterId) - } + } + // Always run metadata sync in refresh loop. Some distributions may not + // bump cluster state version for every index-state change. + if oldIndexStateErr == nil { + module.saveIndexMetadata(state, clusterId) } } if stateChanged { From fc4db22d67728b153808c88f7def7d3ea5101a67 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 12 Jun 2026 18:27:16 +0800 Subject: [PATCH 125/137] fix: type [UnmappedTerms] unsupported --- modules/elastic/adapter/elasticsearch/v0.go | 67 ++++++++++++++++++ .../adapter/elasticsearch/v0_querydsl_test.go | 69 +++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 modules/elastic/adapter/elasticsearch/v0_querydsl_test.go diff --git a/modules/elastic/adapter/elasticsearch/v0.go b/modules/elastic/adapter/elasticsearch/v0.go index 40174c0e0..bd7b0f7c5 100755 --- a/modules/elastic/adapter/elasticsearch/v0.go +++ b/modules/elastic/adapter/elasticsearch/v0.go @@ -621,6 +621,18 @@ func (c *ESAPIV0) QueryDSL(ctx context.Context, indexName string, queryArgs *[]u } resp, err := c.Request(ctx, util.Verb_POST, url, queryDSL) + if err == nil && resp != nil && shouldRetryWithoutTermsMissing(resp.StatusCode, resp.Body) { + if retryQueryDSL, changed := stripTermsMissingFromQueryDSL(queryDSL); changed { + if global.Env().IsDebug { + log.Tracef("retrying query without terms.missing due to UnmappedTerms response: %s", url) + } + retryResp, retryErr := c.Request(ctx, util.Verb_POST, url, retryQueryDSL) + if retryErr == nil && retryResp != nil { + resp = retryResp + queryDSL = retryQueryDSL + } + } + } if resp != nil { esResp.StatusCode = resp.StatusCode esResp.RawResult = resp @@ -647,6 +659,61 @@ func (c *ESAPIV0) QueryDSL(ctx context.Context, indexName string, queryArgs *[]u return esResp, nil } +func shouldRetryWithoutTermsMissing(statusCode int, body []byte) bool { + if statusCode < 500 || len(body) == 0 { + return false + } + lowerBody := strings.ToLower(util.UnsafeBytesToString(body)) + return strings.Contains(lowerBody, "unmappedterms") && + strings.Contains(lowerBody, "unsupported") +} + +func stripTermsMissingFromQueryDSL(queryDSL []byte) ([]byte, bool) { + if len(queryDSL) == 0 { + return nil, false + } + payload := map[string]interface{}{} + if err := json.Unmarshal(queryDSL, &payload); err != nil { + return nil, false + } + changed := stripTermsMissingRecursive(payload) + if !changed { + return nil, false + } + newDSL, err := json.Marshal(payload) + if err != nil { + return nil, false + } + return newDSL, true +} + +func stripTermsMissingRecursive(value interface{}) bool { + changed := false + switch typed := value.(type) { + case map[string]interface{}: + if termsValue, ok := typed["terms"]; ok { + if termsMap, ok := termsValue.(map[string]interface{}); ok { + if _, exists := termsMap["missing"]; exists { + delete(termsMap, "missing") + changed = true + } + } + } + for _, nested := range typed { + if stripTermsMissingRecursive(nested) { + changed = true + } + } + case []interface{}: + for _, nested := range typed { + if stripTermsMissingRecursive(nested) { + changed = true + } + } + } + return changed +} + func (c *ESAPIV0) SearchWithRawQueryDSL(indexName string, queryDSL []byte) (*elastic.SearchResponse, error) { return c.QueryDSL(nil, indexName, nil, queryDSL) } diff --git a/modules/elastic/adapter/elasticsearch/v0_querydsl_test.go b/modules/elastic/adapter/elasticsearch/v0_querydsl_test.go new file mode 100644 index 000000000..b5cc1d430 --- /dev/null +++ b/modules/elastic/adapter/elasticsearch/v0_querydsl_test.go @@ -0,0 +1,69 @@ +package elasticsearch + +import ( + "testing" + + "github.com/segmentio/encoding/json" +) + +func TestStripTermsMissingFromQueryDSL(t *testing.T) { + source := []byte(`{ + "aggs": { + "a": { + "terms": { + "field": "metadata.labels.cluster_id", + "missing": "", + "size": 2 + }, + "aggs": { + "b": { + "date_range": { + "field": "timestamp", + "ranges": [{"from":"now-1d/d","to":"now/d"}] + }, + "aggs": { + "c": { + "terms": { + "field": "payload.elasticsearch.cluster_health.status", + "missing": "", + "size": 2 + } + } + } + } + } + } + } + }`) + + got, changed := stripTermsMissingFromQueryDSL(source) + if !changed { + t.Fatal("expected query DSL to be changed") + } + var parsed map[string]interface{} + if err := json.Unmarshal(got, &parsed); err != nil { + t.Fatalf("expected valid JSON, got %v", err) + } + + aggA := parsed["aggs"].(map[string]interface{})["a"].(map[string]interface{}) + termsA := aggA["terms"].(map[string]interface{}) + if _, ok := termsA["missing"]; ok { + t.Fatalf("expected top-level terms.missing to be removed, got %#v", termsA) + } + + aggB := aggA["aggs"].(map[string]interface{})["b"].(map[string]interface{}) + aggC := aggB["aggs"].(map[string]interface{})["c"].(map[string]interface{}) + termsC := aggC["terms"].(map[string]interface{}) + if _, ok := termsC["missing"]; ok { + t.Fatalf("expected nested terms.missing to be removed, got %#v", termsC) + } +} + +func TestShouldRetryWithoutTermsMissing(t *testing.T) { + if shouldRetryWithoutTermsMissing(400, []byte(`{"error":{"reason":"UnmappedTerms unsupported"}}`)) { + t.Fatal("should not retry on non-5xx status") + } + if !shouldRetryWithoutTermsMissing(500, []byte(`{"error":{"reason":"Aggregation [x] is of type [UnmappedTerms] which is currently unsupported."}}`)) { + t.Fatal("expected retry to be enabled for UnmappedTerms unsupported error") + } +} From 7a305e8feda5ad2cde59526f48525d1f8ec4a3de Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 12 Jun 2026 22:01:55 +0800 Subject: [PATCH 126/137] improve: recovery with bootstrap token for agent --- modules/configs/client/client.go | 60 ++++++++++++++++++++++----- modules/configs/client/client_test.go | 26 ++++++++++++ modules/configs/common/domain.go | 5 ++- 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index d516cde06..6e1a73fbe 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -48,6 +48,7 @@ import ( "infini.sh/framework/core/model" "infini.sh/framework/core/task" "infini.sh/framework/core/util" + ucfg "infini.sh/framework/lib/go-ucfg" "infini.sh/framework/modules/configs/common" "infini.sh/framework/modules/configs/config" ) @@ -61,9 +62,28 @@ var postRegisterHooks []func(server string, res *util.Result) error var unauthorizedRegisterRetryLock sync.Mutex var lastUnauthorizedRegisterRetryAt time.Time var clearManagedRegistrationStateFunc = clearManagedRegistrationState -var reconnectToManagerFunc = func() error { return ConnectToManager() } +var loadManagedBootstrapAccessTokenFunc = func() (string, error) { + return common.LoadTokenFromKeystore(common.ManagerBootstrapTokenKeystoreKey) +} +var restoreManagedBootstrapAccessTokenFunc = func() (string, error) { + token, err := loadManagedBootstrapAccessTokenFunc() + if err != nil { + return "", err + } + token = strings.TrimSpace(token) + if token == "" { + return "", fmt.Errorf("managed bootstrap access token is missing") + } + global.Env().SystemConfig.Configs.ManagerConfig.AccessToken = ucfg.SecretString(token) + return token, nil +} +var reconnectToManagerFunc func() error var configSyncInProgress atomic.Bool +func init() { + reconnectToManagerFunc = ConnectToManager +} + // maskURLInError replaces http(s):// URLs in error messages to avoid leaking internal addresses in logs. func maskURLInError(err error) string { if err == nil { @@ -153,6 +173,12 @@ func ConnectToManager() error { } global.Register(configRegisterEnvKey, true) } else { + if res.StatusCode == http.StatusUnauthorized { + if !claimUnauthorizedRegisterRetrySlot() { + return fmt.Errorf("unauthorized config manager registration") + } + return recoverManagedRegistrationWithBootstrap() + } log.Warnf("config manager registration failed for instance %v via %v: status=%d, body=%s", info.ID, server, res.StatusCode, truncateManagerResponseBodyForLog(res.Body)) return fmt.Errorf("failed to register to config manager: status=%d, body=%s", res.StatusCode, strings.TrimSpace(string(res.Body))) } @@ -223,20 +249,12 @@ func handleUnauthorizedConfigSyncResponse(res *util.Result) bool { return false } - unauthorizedRegisterRetryLock.Lock() - if !lastUnauthorizedRegisterRetryAt.IsZero() && time.Since(lastUnauthorizedRegisterRetryAt) < unauthorizedRegisterRetryInterval { - unauthorizedRegisterRetryLock.Unlock() + if !claimUnauthorizedRegisterRetrySlot() { return true } - lastUnauthorizedRegisterRetryAt = time.Now() - unauthorizedRegisterRetryLock.Unlock() log.Warn("config sync unauthorized, clearing local registration state and retrying registration") - if err := clearManagedRegistrationStateFunc(); err != nil { - log.Warnf("failed to clear local registration state after unauthorized config sync: %v", err) - return true - } - if err := reconnectToManagerFunc(); err != nil { + if err := recoverManagedRegistrationWithBootstrap(); err != nil { log.Warnf("failed to re-register to config manager after unauthorized config sync: %v", err) return true } @@ -244,6 +262,26 @@ func handleUnauthorizedConfigSyncResponse(res *util.Result) bool { return true } +func claimUnauthorizedRegisterRetrySlot() bool { + unauthorizedRegisterRetryLock.Lock() + defer unauthorizedRegisterRetryLock.Unlock() + if !lastUnauthorizedRegisterRetryAt.IsZero() && time.Since(lastUnauthorizedRegisterRetryAt) < unauthorizedRegisterRetryInterval { + return false + } + lastUnauthorizedRegisterRetryAt = time.Now() + return true +} + +func recoverManagedRegistrationWithBootstrap() error { + if _, err := restoreManagedBootstrapAccessTokenFunc(); err != nil { + return err + } + if err := clearManagedRegistrationStateFunc(); err != nil { + return err + } + return reconnectToManagerFunc() +} + func execPostRegisterHooks(server string, res *util.Result) error { for _, hook := range postRegisterHooks { if err := hook(server, res); err != nil { diff --git a/modules/configs/client/client_test.go b/modules/configs/client/client_test.go index aeedc9a22..0a66dab4d 100644 --- a/modules/configs/client/client_test.go +++ b/modules/configs/client/client_test.go @@ -163,15 +163,22 @@ func TestListenConfigChangesStillSyncsAfterHTTPClientInit(t *testing.T) { func TestHandleUnauthorizedConfigSyncResponseClearsStateAndReconnects(t *testing.T) { oldClear := clearManagedRegistrationStateFunc oldReconnect := reconnectToManagerFunc + oldLoadBootstrap := loadManagedBootstrapAccessTokenFunc + oldRestoreBootstrap := restoreManagedBootstrapAccessTokenFunc oldRetryAt := lastUnauthorizedRegisterRetryAt + oldAccessToken := global.Env().SystemConfig.Configs.ManagerConfig.AccessToken t.Cleanup(func() { clearManagedRegistrationStateFunc = oldClear reconnectToManagerFunc = oldReconnect + loadManagedBootstrapAccessTokenFunc = oldLoadBootstrap + restoreManagedBootstrapAccessTokenFunc = oldRestoreBootstrap lastUnauthorizedRegisterRetryAt = oldRetryAt + global.Env().SystemConfig.Configs.ManagerConfig.AccessToken = oldAccessToken }) var cleared atomic.Int32 var reconnected atomic.Int32 + var restored atomic.Int32 clearManagedRegistrationStateFunc = func() error { cleared.Add(1) return nil @@ -180,18 +187,37 @@ func TestHandleUnauthorizedConfigSyncResponseClearsStateAndReconnects(t *testing reconnected.Add(1) return nil } + loadManagedBootstrapAccessTokenFunc = func() (string, error) { + restored.Add(1) + return "bootstrap-token", nil + } + restoreManagedBootstrapAccessTokenFunc = func() (string, error) { + token, err := loadManagedBootstrapAccessTokenFunc() + if err != nil { + return "", err + } + global.Env().SystemConfig.Configs.ManagerConfig.AccessToken = ucfg.SecretString(token) + return token, nil + } lastUnauthorizedRegisterRetryAt = time.Time{} + global.Env().SystemConfig.Configs.ManagerConfig.AccessToken = "" handled := handleUnauthorizedConfigSyncResponse(&util.Result{StatusCode: http.StatusUnauthorized}) if !handled { t.Fatal("expected unauthorized config sync response to be handled") } + if restored.Load() != 1 { + t.Fatalf("expected bootstrap token to be loaded once, got %d", restored.Load()) + } if cleared.Load() != 1 { t.Fatalf("expected local registration state to be cleared once, got %d", cleared.Load()) } if reconnected.Load() != 1 { t.Fatalf("expected reconnect to run once, got %d", reconnected.Load()) } + if got := global.Env().SystemConfig.Configs.ManagerConfig.AccessToken.Get(); got != "bootstrap-token" { + t.Fatalf("expected bootstrap token to be restored, got %q", got) + } handled = handleUnauthorizedConfigSyncResponse(&util.Result{StatusCode: http.StatusUnauthorized}) if !handled { diff --git a/modules/configs/common/domain.go b/modules/configs/common/domain.go index bb5780c09..23faabba1 100644 --- a/modules/configs/common/domain.go +++ b/modules/configs/common/domain.go @@ -37,8 +37,9 @@ const REGISTER_API = "/instance/_register" const SYNC_API = "/configs/_sync" const ( - ManagerTokenKeystoreKey = "configs_manager_token" - AgentAccessTokenKeystoreKey = "agent_access_token" + ManagerTokenKeystoreKey = "configs_manager_token" + ManagerBootstrapTokenKeystoreKey = "configs_manager_bootstrap_token" + AgentAccessTokenKeystoreKey = "agent_access_token" ) type RegisterToken struct { From 7cde0619167268d4a737ad204ede8b63e6455604 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 13 Jun 2026 19:15:34 +0800 Subject: [PATCH 127/137] chore: temp file for target file was exits, skip --- modules/queue/disk_queue/compress.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/queue/disk_queue/compress.go b/modules/queue/disk_queue/compress.go index 63404a21e..451e66e19 100644 --- a/modules/queue/disk_queue/compress.go +++ b/modules/queue/disk_queue/compress.go @@ -34,6 +34,7 @@ import ( "infini.sh/framework/core/util" "infini.sh/framework/core/util/zstd" "os" + "strings" "sync" ) @@ -124,6 +125,10 @@ func (module *DiskQueue) compressFiles(queueID string, fileNum int64) { //compress err := zstd.CompressFile(file, toFile) if err != nil { + if strings.Contains(err.Error(), "temp file for target file was exits, skip:") { + log.Debug(err) + continue + } log.Error(err) continue } From 3771157b54c0c56111e41e0916816f6d2f1f1332 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 15 Jun 2026 15:27:45 +0800 Subject: [PATCH 128/137] improve: update the host avaliable check --- core/elastic/actions.go | 24 +++++++++++++---- core/elastic/actions_test.go | 18 +++++++++++++ modules/elastic/metadata.go | 33 ++++++++++++++++++++++++ modules/elastic/module.go | 10 ++++--- modules/metrics/elastic/elasticsearch.go | 23 +++++++++++++++++ 5 files changed, 100 insertions(+), 8 deletions(-) diff --git a/core/elastic/actions.go b/core/elastic/actions.go index 9401a3a14..1eb11e5ef 100644 --- a/core/elastic/actions.go +++ b/core/elastic/actions.go @@ -159,6 +159,16 @@ func (meta *ElasticsearchMetadata) shouldTraceUnavailableReason() bool { return !meta.Config.Monitored } +func (meta *ElasticsearchMetadata) shouldCheckActiveHostsOnFailure() bool { + if meta == nil || meta.Config == nil { + return true + } + if meta.Config.MetadataConfigs != nil && !meta.Config.MetadataConfigs.NodeAvailabilityCheck.Enabled { + return false + } + return true +} + func (meta *ElasticsearchMetadata) Init(health bool) { meta.clusterAvailable = health if health && meta.Health == nil { @@ -455,11 +465,15 @@ func (meta *ElasticsearchMetadata) ReportFailure(errorMessage error) bool { return true } - num := meta.GetActiveHosts() - log.Infof("%v has active hosts: %v", meta.Config.Name, num) - if num > 0 { - log.Debugf("enough failure ticket for elasticsearch [%v], but still have [%v] alive nodes", meta.Config.Name, num) - return false + if meta.shouldCheckActiveHostsOnFailure() { + num := meta.GetActiveHosts() + log.Infof("%v has active hosts: %v", meta.Config.Name, num) + if num > 0 { + log.Debugf("enough failure ticket for elasticsearch [%v], but still have [%v] alive nodes", meta.Config.Name, num) + return false + } + } else if rate.GetRateLimiter("cluster_active_hosts_check", meta.Config.Name, 1, 1, 30*time.Second).Allow() { + log.Infof("skip active hosts check for elasticsearch [%v], node availability check is disabled", meta.Config.Name) } log.Debugf("enough failure ticket for elasticsearch [%v], mark it down", meta.Config.Name) diff --git a/core/elastic/actions_test.go b/core/elastic/actions_test.go index 8b155fbe1..5d706b190 100644 --- a/core/elastic/actions_test.go +++ b/core/elastic/actions_test.go @@ -143,3 +143,21 @@ func TestShouldTraceUnavailableReasonForUnmonitoredCluster(t *testing.T) { t.Fatal("expected monitored cluster to keep debug unavailable reason") } } + +func TestShouldCheckActiveHostsOnFailure(t *testing.T) { + meta := &ElasticsearchMetadata{Config: &ElasticsearchConfig{}} + if !meta.shouldCheckActiveHostsOnFailure() { + t.Fatal("expected active hosts check enabled by default") + } + + meta.Config.MetadataConfigs = &MetadataConfig{} + meta.Config.MetadataConfigs.NodeAvailabilityCheck.Enabled = true + if !meta.shouldCheckActiveHostsOnFailure() { + t.Fatal("expected active hosts check enabled when node availability check is on") + } + + meta.Config.MetadataConfigs.NodeAvailabilityCheck.Enabled = false + if meta.shouldCheckActiveHostsOnFailure() { + t.Fatal("expected active hosts check disabled when node availability check is off") + } +} diff --git a/modules/elastic/metadata.go b/modules/elastic/metadata.go index f6bef05c1..702edc39b 100644 --- a/modules/elastic/metadata.go +++ b/modules/elastic/metadata.go @@ -252,6 +252,7 @@ func SyncClusterHealthStatus(clusterID string) { // update cluster state, on state version change func (module *ElasticModule) updateClusterState(clusterId string, force bool) { + startAt := time.Now() meta := elastic.GetMetadata(clusterId) if meta == nil { @@ -265,6 +266,12 @@ func (module *ElasticModule) updateClusterState(clusterId string, force bool) { return } + interval := moduleConfig.MetadataRefresh.Interval + if meta.Config != nil && meta.Config.MetadataConfigs != nil && meta.Config.MetadataConfigs.MetadataRefresh.Interval != "" { + interval = meta.Config.MetadataConfigs.MetadataRefresh.Interval + } + intervalD := util.GetDurationOrDefault(interval, 30*time.Second) + client := elastic.GetClient(clusterId) state, err := client.GetClusterState() if err != nil { @@ -277,6 +284,18 @@ func (module *ElasticModule) updateClusterState(clusterId string, force bool) { } if state != nil { + responseSize := uint64(0) + if state.RawResult != nil { + responseSize = state.RawResult.Size + } + indexCount := 0 + if state.Metadata != nil { + indexCount = len(state.Metadata.Indices) + } + routingIndexCount := 0 + if state.RoutingTable != nil { + routingIndexCount = len(state.RoutingTable.Indices) + } stateChanged := false if meta.ClusterState == nil { stateChanged = true @@ -314,6 +333,20 @@ func (module *ElasticModule) updateClusterState(clusterId string, force bool) { state.Metadata = metaData meta.ClusterState = state } + elapsed := time.Since(startAt) + if elapsed > intervalD { + log.Warnf( + "refresh cluster state for cluster [%s] completed slowly, elapsed: %v, interval: %s, response_size: %d bytes, compressed_size_in_bytes: %d, metadata_indices: %d, routing_indices: %d, state_version: %d", + meta.Config.Name, + elapsed, + interval, + responseSize, + state.CompressedSizeInBytes, + indexCount, + routingIndexCount, + state.Version, + ) + } } } diff --git a/modules/elastic/module.go b/modules/elastic/module.go index 84cd9902e..59f3cbd5d 100755 --- a/modules/elastic/module.go +++ b/modules/elastic/module.go @@ -281,6 +281,7 @@ func nodeAvailabilityCheck() { } if time.Since(startTime.(time.Time)) > util.GetDurationOrDefault(interval, 10*time.Second)*2 { log.Warnf("check availability for node [%s] is still running, elapsed: %v, skip waiting", v.Host, elapsed.String()) + return true } else { log.Warnf("check availability for node [%s] is still running, elapsed: %v", v.Host, elapsed.String()) return true @@ -342,6 +343,7 @@ func (module *ElasticModule) registerClusterStateRefreshTask() { intervalD := util.GetDurationOrDefault(interval, 10*time.Second) if time.Since(startTime.(time.Time)) > intervalD*2 { log.Warnf("refresh cluster state for cluster [%s] is still running, elapsed: %v, skip waiting", v.Name, elapsed.String()) + return true } else { duration := elapsed - intervalD abd := math.Abs(duration.Seconds()) @@ -355,8 +357,8 @@ func (module *ElasticModule) registerClusterStateRefreshTask() { task.RunWithContext("refresh_cluster_state", func(ctx context.Context) error { clusterID := task.MustGetString(ctx, "id") + defer module.stateMap.Delete(clusterID) module.updateClusterState(clusterID, false) - module.stateMap.Delete(clusterID) return nil }, context.WithValue(context.Background(), "id", v.ID)) } @@ -524,6 +526,7 @@ func (module *ElasticModule) Start() error { tinterval := util.GetDurationOrDefault(interval, 10*time.Second) if elapsed > tinterval*2 { log.Warnf("health check for cluster [%s] is still running, elapsed: %v, skip waiting", cfg1.Name, elapsed.String()) + return true } else if math.Abs((elapsed - tinterval).Seconds()) > 3 { log.Warnf("health check for cluster [%s] is still running, elapsed: %v", cfg1.Name, elapsed.String()) return true @@ -533,8 +536,8 @@ func (module *ElasticModule) Start() error { task.RunWithContext("refresh_cluster_health", func(ctx context.Context) error { clusterID := task.MustGetString(ctx, "id") + defer module.healthMap.Delete(clusterID) module.clusterHealthCheck(clusterID, false) - module.healthMap.Delete(clusterID) return nil }, context.WithValue(context.Background(), "id", cfg1.ID)) } @@ -691,6 +694,7 @@ func (module *ElasticModule) registerClusterSettingsRefreshTask() { if time.Since(startTime.(time.Time)) > util.GetDurationOrDefault(interval, 10*time.Second)*2 { log.Warnf("collect cluster settings for cluster [%s] is still running, elapsed: %v, skip waiting", v.Name, elapsed.String()) + return true } else { log.Warnf("collect cluster settings for cluster [%s] is still running, elapsed: %v", v.Name, elapsed.String()) return true @@ -699,8 +703,8 @@ func (module *ElasticModule) registerClusterSettingsRefreshTask() { module.settingsMap.Store(v.ID, time.Now()) task.RunWithContext("refresh_cluster_settings", func(ctx context.Context) error { clusterID := task.MustGetString(ctx, "id") + defer module.settingsMap.Delete(clusterID) module.updateClusterSettings(clusterID) - module.settingsMap.Delete(clusterID) return nil }, context.WithValue(context.Background(), "id", v.ID)) } diff --git a/modules/metrics/elastic/elasticsearch.go b/modules/metrics/elastic/elasticsearch.go index ac8d3bfa5..b5f27b12a 100644 --- a/modules/metrics/elastic/elasticsearch.go +++ b/modules/metrics/elastic/elasticsearch.go @@ -653,6 +653,7 @@ func (m *ElasticsearchMetric) CollectClusterHealth(k string, v *elastic.Elastics func (m *ElasticsearchMetric) CollectClusterState(k string, v *elastic.ElasticsearchMetadata) error { log.Trace("collecting custer state metrics for :", k) + startAt := time.Now() client := elastic.GetClient(k) @@ -672,6 +673,28 @@ func (m *ElasticsearchMetric) CollectClusterState(k string, v *elastic.Elasticse if err != nil { return wrapMetricCollectError(v.Config.Name, "cluster_stats", v.Config.GetAnyEndpoint(), monitorCfg.ClusterStats.Interval, err) } + elapsed := time.Since(startAt) + if elapsed > du*8/10 { + responseSize := uint64(0) + if stats != nil && stats.RawResult != nil { + responseSize = stats.RawResult.Size + } + indexFieldCount := 0 + nodeFieldCount := 0 + if stats != nil { + indexFieldCount = len(stats.Indices) + nodeFieldCount = len(stats.Nodes) + } + log.Warnf( + "collect cluster_stats for cluster [%s] is near timeout, elapsed: %v, timeout: %s, response_size: %d bytes, index_fields: %d, node_fields: %d", + v.Config.Name, + elapsed, + monitorCfg.ClusterStats.Interval, + responseSize, + indexFieldCount, + nodeFieldCount, + ) + } item := event.Event{ Metadata: event.EventMetadata{ From fefc4e10f6f6b43fac7b932de696e726523d148f Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 15 Jun 2026 16:20:02 +0800 Subject: [PATCH 129/137] improve: restore gateway register to console --- modules/configs/client/client.go | 10 ++++++ modules/configs/client/client_test.go | 48 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/modules/configs/client/client.go b/modules/configs/client/client.go index 6e1a73fbe..1a6b9e0ec 100644 --- a/modules/configs/client/client.go +++ b/modules/configs/client/client.go @@ -71,6 +71,16 @@ var restoreManagedBootstrapAccessTokenFunc = func() (string, error) { return "", err } token = strings.TrimSpace(token) + if token == "" { + token = strings.TrimSpace(global.Env().SystemConfig.Configs.ManagerConfig.AccessToken.Get()) + } + if token == "" { + token, err = common.LoadTokenFromKeystore(common.ManagerTokenKeystoreKey) + if err != nil { + return "", err + } + token = strings.TrimSpace(token) + } if token == "" { return "", fmt.Errorf("managed bootstrap access token is missing") } diff --git a/modules/configs/client/client_test.go b/modules/configs/client/client_test.go index 0a66dab4d..965417018 100644 --- a/modules/configs/client/client_test.go +++ b/modules/configs/client/client_test.go @@ -251,3 +251,51 @@ func TestManagedConfigSyncGuardPreventsOverlap(t *testing.T) { } finishManagedConfigSync() } + +func TestRestoreManagedBootstrapAccessTokenFallsBackToManagerAccessToken(t *testing.T) { + t.Setenv("KEYSTORE_PATH", t.TempDir()) + + oldLoadBootstrap := loadManagedBootstrapAccessTokenFunc + oldAccessToken := global.Env().SystemConfig.Configs.ManagerConfig.AccessToken + t.Cleanup(func() { + loadManagedBootstrapAccessTokenFunc = oldLoadBootstrap + global.Env().SystemConfig.Configs.ManagerConfig.AccessToken = oldAccessToken + }) + + loadManagedBootstrapAccessTokenFunc = func() (string, error) { + return "", nil + } + global.Env().SystemConfig.Configs.ManagerConfig.AccessToken = ucfg.SecretString("config-access-token") + + token, err := restoreManagedBootstrapAccessTokenFunc() + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + if token != "config-access-token" { + t.Fatalf("expected config access token fallback, got %q", token) + } +} + +func TestRestoreManagedBootstrapAccessTokenReturnsErrorWhenNoFallbackAvailable(t *testing.T) { + t.Setenv("KEYSTORE_PATH", t.TempDir()) + + oldLoadBootstrap := loadManagedBootstrapAccessTokenFunc + oldAccessToken := global.Env().SystemConfig.Configs.ManagerConfig.AccessToken + t.Cleanup(func() { + loadManagedBootstrapAccessTokenFunc = oldLoadBootstrap + global.Env().SystemConfig.Configs.ManagerConfig.AccessToken = oldAccessToken + }) + + loadManagedBootstrapAccessTokenFunc = func() (string, error) { + return "", nil + } + global.Env().SystemConfig.Configs.ManagerConfig.AccessToken = "" + + _, err := restoreManagedBootstrapAccessTokenFunc() + if err == nil { + t.Fatal("expected missing bootstrap token error") + } + if !strings.Contains(err.Error(), "managed bootstrap access token is missing") { + t.Fatalf("unexpected error: %v", err) + } +} From 1ff449212a77ed1480a4c3598985c42d544f12c4 Mon Sep 17 00:00:00 2001 From: medcl Date: Tue, 16 Jun 2026 09:33:26 +0800 Subject: [PATCH 130/137] chore: add interval to bucket --- core/elastic/index.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/core/elastic/index.go b/core/elastic/index.go index 27768169e..bdb034440 100755 --- a/core/elastic/index.go +++ b/core/elastic/index.go @@ -216,14 +216,16 @@ type Bucket struct { } type AggregationResponse struct { - Buckets []BucketBase `json:"buckets,omitempty"` - Value interface{} `json:"value,omitempty"` + Buckets []BucketBase `json:"buckets,omitempty"` + Value interface{} `json:"value,omitempty"` + Interval string `json:"interval,omitempty"` } func (a *AggregationResponse) UnmarshalJSON(data []byte) error { type alias struct { - Buckets json.RawMessage `json:"buckets,omitempty"` - Value interface{} `json:"value,omitempty"` + Buckets json.RawMessage `json:"buckets,omitempty"` + Value interface{} `json:"value,omitempty"` + Interval string `json:"interval,omitempty"` } var aux alias @@ -231,6 +233,7 @@ func (a *AggregationResponse) UnmarshalJSON(data []byte) error { return err } a.Value = aux.Value + a.Interval = aux.Interval buckets := bytes.TrimSpace(aux.Buckets) if len(buckets) == 0 || bytes.Equal(buckets, []byte("null")) { From d88324b0b830ac8774dd16960da118e8673e235b Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 16 Jun 2026 16:42:46 +0800 Subject: [PATCH 131/137] improve: add get all endpoints --- core/elastic/domain_actions.go | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/core/elastic/domain_actions.go b/core/elastic/domain_actions.go index 90e84f388..d343ba72f 100644 --- a/core/elastic/domain_actions.go +++ b/core/elastic/domain_actions.go @@ -208,6 +208,44 @@ func (c *ElasticsearchConfig) GetAnyEndpoint() string { panic(fmt.Errorf("no endpoint was not found in config [%v] ", c.ID)) } +func (c *ElasticsearchConfig) GetAllEndpoints() []string { + build := func(host string) string { + return fmt.Sprintf("%s://%s", c.Schema, host) + } + + seen := make(map[string]struct{}) + result := make([]string, 0) + + add := func(v string) { + if v == "" { + return + } + if _, ok := seen[v]; ok { + return + } + seen[v] = struct{}{} + result = append(result, v) + } + + // 1. Hosts -> schema + host + for _, host := range c.Hosts { + add(build(host)) + } + + // 2. Endpoints -> raw + for _, ep := range c.Endpoints { + add(ep) + } + + // 3. Endpoint -> raw single + add(c.Endpoint) + + // 4. Host -> schema + host + add(build(c.Host)) + + return result +} + func (meta *ElasticsearchMetadata) GetMajorVersion() int { versionLock.RLock() From 8e5a12fa34ddaae53bdbeb94e00ce19364dc8df9 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 16 Jun 2026 17:11:55 +0800 Subject: [PATCH 132/137] fix: element for array with quote --- core/elastic/domain_actions.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/core/elastic/domain_actions.go b/core/elastic/domain_actions.go index d343ba72f..634246a08 100644 --- a/core/elastic/domain_actions.go +++ b/core/elastic/domain_actions.go @@ -43,6 +43,7 @@ import ( "crypto/tls" "fmt" uri "net/url" + "strconv" "strings" "sync" "time" @@ -216,15 +217,25 @@ func (c *ElasticsearchConfig) GetAllEndpoints() []string { seen := make(map[string]struct{}) result := make([]string, 0) + quote := func(v string) string { + if v == "" { + return "" + } + return strconv.Quote(v) + } + add := func(v string) { if v == "" { return } - if _, ok := seen[v]; ok { + + qv := quote(v) + + if _, ok := seen[qv]; ok { return } - seen[v] = struct{}{} - result = append(result, v) + seen[qv] = struct{}{} + result = append(result, qv) } // 1. Hosts -> schema + host From cc62257d298233f902ae1ea708c6b0bcc58b75a6 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 11 Jul 2026 11:21:20 +0800 Subject: [PATCH 133/137] fix: add event sink for metric --- modules/metrics/host/overall/overall.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/metrics/host/overall/overall.go b/modules/metrics/host/overall/overall.go index 81fed49c0..074980988 100644 --- a/modules/metrics/host/overall/overall.go +++ b/modules/metrics/host/overall/overall.go @@ -48,6 +48,7 @@ type Metric struct { IntervalSeconds float64 `config:"interval_seconds"` YellowThreshold float64 `config:"yellow_threshold"` RedThreshold float64 `config:"red_threshold"` + event.EventSink mu sync.Mutex @@ -99,6 +100,11 @@ type deviceUtilization struct { } func New(cfg *config.Config) (*Metric, error) { + return NewWithSink(cfg, event.DefaultEventSink) +} + +// NewWithSink creates an overall metric collector with a custom sink. +func NewWithSink(cfg *config.Config, sink event.EventSink) (*Metric, error) { me := &Metric{ Enabled: true, IntervalSeconds: 10, @@ -108,6 +114,7 @@ func New(cfg *config.Config) (*Metric, error) { prevNetIO: make(map[string]*netIOSnapshot), netBandwidth: make(map[string]float64), } + me.EventSink = sink err := cfg.Unpack(&me) if err != nil { @@ -235,7 +242,7 @@ func (m *Metric) Collect() error { fields["status"] = status fields["bottleneck"] = bottleneck - return event.Save(&event.Event{ + return m.Save(&event.Event{ Metadata: event.EventMetadata{ Category: "host", Name: "overall", From 12b16a19feea7dd70bfacdf37be3627c29694982 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 29 Jul 2026 14:44:24 +0800 Subject: [PATCH 134/137] chore: code format and audit with user and role --- core/env/env.go | 2 +- core/env/env_test.go | 2 +- core/event/store.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/env/env.go b/core/env/env.go index 79f32f960..45b61ff8e 100755 --- a/core/env/env.go +++ b/core/env/env.go @@ -845,4 +845,4 @@ func (env *Env) UpdateState(i int32) { func (env *Env) GetState() int32 { return atomic.LoadInt32(&env.state) -} \ No newline at end of file +} diff --git a/core/env/env_test.go b/core/env/env_test.go index da9444a4a..23e00d752 100644 --- a/core/env/env_test.go +++ b/core/env/env_test.go @@ -115,4 +115,4 @@ func TestParseConfigSection_KeyExistsButPrimitive_ReturnsError(t *testing.T) { assert.False(t, exist) require.Error(t, err) -} \ No newline at end of file +} diff --git a/core/event/store.go b/core/event/store.go index 067893803..e5634b778 100644 --- a/core/event/store.go +++ b/core/event/store.go @@ -113,4 +113,4 @@ func SaveLog(event *Event) error { } return nil -} \ No newline at end of file +} From bf1de8187bb966d0bb4b2c9e4f82b2316fc29b7f Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 7 Aug 2026 23:33:04 +0800 Subject: [PATCH 135/137] fix: api token with api --- core/env/env.go | 10 +++++++--- core/env/env_test.go | 8 ++++++++ modules/security/access_token/authentication.go | 3 +-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/core/env/env.go b/core/env/env.go index 45b61ff8e..68b5df3f0 100755 --- a/core/env/env.go +++ b/core/env/env.go @@ -294,9 +294,13 @@ func GetDefaultSystemConfig() config.SystemConfig { }, Security: config.WebSecurityConfig{ Enabled: true, - Authentication: config.AuthenticationConfig{Native: config.RealmConfig{ - Enabled: false, - }, + Authentication: config.AuthenticationConfig{ + Native: config.RealmConfig{ + Enabled: false, + }, + AccessToken: config.AccessTokenConfig{ + Enabled: true, + }, }, }, WebsocketConfig: config.WebsocketConfig{ diff --git a/core/env/env_test.go b/core/env/env_test.go index 23e00d752..74afd9f2e 100644 --- a/core/env/env_test.go +++ b/core/env/env_test.go @@ -31,6 +31,14 @@ import ( "infini.sh/framework/core/config" ) +func TestGetDefaultSystemConfigEnablesAccessTokenAPI(t *testing.T) { + cfg := GetDefaultSystemConfig() + + if !cfg.WebAppConfig.Security.Authentication.AccessToken.Enabled { + t.Fatal("expected access token api to be enabled by default") + } +} + func TestParseConfigSection_NilConfig(t *testing.T) { var out struct{ Foo string } exist, err := ParseConfigSection(nil, "anykey", &out) diff --git a/modules/security/access_token/authentication.go b/modules/security/access_token/authentication.go index a1c860ee8..e6701094b 100644 --- a/modules/security/access_token/authentication.go +++ b/modules/security/access_token/authentication.go @@ -22,7 +22,6 @@ import ( "infini.sh/framework/core/orm" "infini.sh/framework/core/security" "infini.sh/framework/core/util" - "infini.sh/framework/modules/security/http_filters" ) const ProviderName = "access_token" @@ -68,7 +67,7 @@ func init() { security.RegisterHTTPAuthFilterProviderWithPriority("api_token", byAPITokenHeader, 30) api.HandleUIMethod(api.POST, "/auth/access_token", RequestAccessToken, api.RequirePermission(createTokenPermission)) - api.HandleUIMethod(api.GET, "/auth/access_token/_search", SearchAccessToken, api.RequirePermission(searchTokenPermission), api.Feature(http_filters.FeatureMaskSensitiveField)) + api.HandleUIMethod(api.GET, "/auth/access_token/_search", SearchAccessToken, api.RequirePermission(searchTokenPermission)) api.HandleUIMethod(api.DELETE, "/auth/access_token/:token_id", DeleteAccessToken, api.RequirePermission(deleteTokenPermission)) api.HandleUIMethod(api.PUT, "/auth/access_token/:token_id", UpdateAccessToken, api.RequirePermission(updateTokenPermission)) From 1cb5787f7acb480d25ae10c5ba1a4bb41e0c8591 Mon Sep 17 00:00:00 2001 From: hardy Date: Sat, 8 Aug 2026 11:46:08 +0800 Subject: [PATCH 136/137] fix: token with api expire date --- .../security/access_token/authentication.go | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/modules/security/access_token/authentication.go b/modules/security/access_token/authentication.go index e6701094b..6c0808778 100644 --- a/modules/security/access_token/authentication.go +++ b/modules/security/access_token/authentication.go @@ -26,6 +26,8 @@ import ( const ProviderName = "access_token" +const defaultAPITokenTTL = 365 * 24 * time.Hour + const ( // KVAccessTokenBucket stores token_string -> AccessToken JSON. Used by // byAPITokenHeader to authenticate inbound requests in both modes. @@ -171,6 +173,7 @@ func RequestAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter. reqBody := struct { Name string `json:"name"` Description string `json:"description"` + ExpireIn *int64 `json:"expire_in,omitempty"` Permissions []security.PermissionKey `json:"permissions,omitempty"` }{} err = api.DecodeJSON(req, &reqBody) @@ -201,7 +204,10 @@ func RequestAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter. } } - expiredAT := time.Now().Add(365 * 24 * time.Hour).Unix() + expiredAT, err := normalizeAPITokenExpireAt(reqBody.ExpireIn) + if err != nil { + panic(errors.ErrorWithHTTPCode(err, 400, "invalid expire_in")) + } res, err := CreateAPIToken(reqUser, reqBody.Name, reqBody.Description, "general", expiredAT, permissions) if err != nil { panic(err) @@ -409,6 +415,7 @@ func UpdateAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter.P reqBody := struct { Name string `json:"name,omitempty"` Description string `json:"description"` + ExpireIn *int64 `json:"expire_in,omitempty"` Permissions []security.PermissionKey `json:"permissions,omitempty"` }{} err = api.DecodeJSON(req, &reqBody) @@ -458,6 +465,13 @@ func UpdateAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter.P if reqBody.Description != "" { token.Description = reqBody.Description } + if reqBody.ExpireIn != nil { + expiredAT, err := normalizeAPITokenExpireAt(reqBody.ExpireIn) + if err != nil { + panic(errors.ErrorWithHTTPCode(err, 400, "invalid expire_in")) + } + token.ExpireIn = expiredAT + } if len(reqBody.Permissions) > 0 { if isNative() { @@ -490,6 +504,19 @@ func UpdateAccessToken(w http.ResponseWriter, req *http.Request, ps httprouter.P api.WriteUpdatedOKJSON(w, tokenID) } +func normalizeAPITokenExpireAt(expireIn *int64) (int64, error) { + if expireIn == nil { + return time.Now().Add(defaultAPITokenTTL).Unix(), nil + } + if *expireIn <= 0 { + return 0, nil + } + if *expireIn <= time.Now().Unix() { + return 0, errors.Errorf("expire_in must be greater than current time") + } + return *expireIn, nil +} + // GenerateApiTokenName generates a unique API token name func GenerateApiTokenName(prefix string) string { if prefix == "" { From 5e3b781a8627d543c0fae534db08e2527c56f6f5 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 19 Aug 2026 17:55:32 +0800 Subject: [PATCH 137/137] fix: reset password and secrets --- core/credential/credential.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/credential/credential.go b/core/credential/credential.go index 058a8379c..701ab99e1 100644 --- a/core/credential/credential.go +++ b/core/credential/credential.go @@ -83,7 +83,7 @@ func (cred *Credential) DecodeBasicAuth() (*model.BasicAuth, error) { var dv interface{} dv, err := cred.Decode() if err != nil { - panic(err) + return nil, err } if auth, ok := dv.(model.BasicAuth); ok {