Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/api/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ var (
ErrFailedToGetSubscription = errors.New("failed to get subscription data")
ErrFailedToUpdateSubscription = errors.New("failed to update subscription")
ErrFailedToGetQueryLogs = errors.New("failed to get profile query logs")
ErrFailedToGetQueryLogDevices = errors.New("failed to get profile query log devices")
ErrFailedToGetStatistics = errors.New("failed to get profile statistics")
ErrFailedToDeleteQueryLogs = errors.New("failed to delete profile query logs")
ErrFailedToGetAccount = errors.New("failed to get account data")
Expand Down
26 changes: 26 additions & 0 deletions api/api/query_logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,32 @@ func (s *APIServer) getProfileQueryLogs() fiber.Handler {
return handler
}

// @Summary Get profile query log devices
// @Description List distinct device IDs seen in the profile's query logs (current retention window), each with its last-seen timestamp, sorted by device ID
// @Tags QueryLogs
// @Produce json
// @Security ApiKeyAuth
// @Param id path string true "Profile ID"
// @Success 200 {object} []model.QueryLogDevice
// @Failure 404 {object} ErrResponse
// @Failure 429 {object} ErrResponse
// @Failure 500 {object} ErrResponse
// @Router /api/v1/profiles/{id}/logs/devices [get]
func (s *APIServer) getProfileQueryLogDevices() fiber.Handler {
handler := func(c *fiber.Ctx) error {
profileId := c.Params("id")
accountId := auth.GetAccountID(c)
devices, err := s.Service.GetProfileQueryLogDevices(c.UserContext(), accountId, profileId)
if err != nil {
log.Ctx(c.UserContext()).Error().Err(err).Msg(ErrFailedToGetQueryLogDevices.Error())
return HandleError(c, err, ErrFailedToGetQueryLogDevices.Error())
}

return c.Status(200).JSON(devices)
}
return handler
}

// @Summary Download profile query logs
// @Description Download profile query logs
// @Tags QueryLogs
Expand Down
1 change: 1 addition & 0 deletions api/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,7 @@ func (s *APIServer) RegisterRoutes() {

// Query logs endpoints
profiles.Get("/:id/logs", middleware.NewLimit(500, 1*time.Minute), s.getProfileQueryLogs())
profiles.Get("/:id/logs/devices", middleware.NewLimit(20, 1*time.Minute), s.getProfileQueryLogDevices())
profiles.Get("/:id/logs/download", middleware.NewLimit(20, 1*time.Minute), s.downloadProfileQueryLogs())
profiles.Delete("/:id/logs", middleware.NewLimit(20, 1*time.Minute), s.deleteProfileQueryLogs())

Expand Down
2 changes: 1 addition & 1 deletion api/db/mongodb/migrations/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ https://pkg.go.dev/github.com/golang-migrate/migrate/v4/database/mongodb#section

### Query logs collections

Note: Query logs time series collections and their indexes creation is currently handled by proxy service.
Note: Query logs time-series collections are created by the proxy service. Their only index is the `{profile_id, timestamp}` meta+time index MongoDB creates automatically on time-series creation (≥6.3) — no code creates query-log indexes explicitly (verified against prod, moddns-shadow#688).
47 changes: 47 additions & 0 deletions api/db/mongodb/query_logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,53 @@ func (r *QueryLogsRepository) GetQueryLogs(ctx context.Context, profileId string
return results, nil
}

// GetQueryLogDevices returns the distinct non-empty device IDs present in the
// profile's query logs (current retention collection, whole window — the TTL
// already bounds it), each with its most recent timestamp, sorted by device ID.
// Aggregation, not Collection.Distinct(): the distinct command is unsupported
// on time-series collections.
func (r *QueryLogsRepository) GetQueryLogDevices(ctx context.Context, profileId string, retention model.Retention) ([]model.QueryLogDevice, error) {
start := time.Now()
coll := r.getCollObject(retention)

pipeline := mongo.Pipeline{
bson.D{primitive.E{Key: "$match", Value: bson.D{
primitive.E{Key: "profile_id", Value: profileId},
// $nin (not $ne "") also excludes docs where the field is missing.
primitive.E{Key: "device_id", Value: bson.D{{Key: "$nin", Value: bson.A{"", nil}}}},
}}},
bson.D{primitive.E{Key: "$group", Value: bson.D{
primitive.E{Key: "_id", Value: "$device_id"},
primitive.E{Key: "last_seen", Value: bson.D{{Key: "$max", Value: "$timestamp"}}},
}}},
bson.D{primitive.E{Key: "$sort", Value: bson.D{{Key: "_id", Value: 1}}}},
// Safety valve against pathological device counts; ids are ≤36 chars.
bson.D{primitive.E{Key: "$limit", Value: 500}},
}

cursor, err := coll.Aggregate(ctx, pipeline)
if err != nil {
return nil, err
}

results := make([]model.QueryLogDevice, 0)
if err = cursor.All(ctx, &results); err != nil {
return nil, err
}
duration := time.Since(start)
if duration > slowQueryThreshold {
// Counts/durations only — device IDs are sensitive log keys.
log.Ctx(ctx).Warn().
Bool("slow", true).
Str("retention", string(retention)).
Int("result_count", len(results)).
Dur("duration", duration).
Msg("Query log devices fetch took too long")
}

return results, nil
}

func buildSortSpec(sortBy string) bson.D {
switch sortBy {
case "domain":
Expand Down
1 change: 1 addition & 0 deletions api/db/repository/query_logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,6 @@ import (

type QueryLogsRepository interface {
GetQueryLogs(ctx context.Context, profileId string, retention model.Retention, status string, timespan int, deviceId, search, sortBy string, page, limit int) ([]model.QueryLog, error)
GetQueryLogDevices(ctx context.Context, profileId string, retention model.Retention) ([]model.QueryLogDevice, error)
DeleteQueryLogs(ctx context.Context, profileId string) error
}
66 changes: 66 additions & 0 deletions api/docs/docs.go
Original file line number Diff line number Diff line change
Expand Up @@ -1891,6 +1891,61 @@ const docTemplate = `{
}
}
},
"/api/v1/profiles/{id}/logs/devices": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List distinct device IDs seen in the profile's query logs (current retention window), each with its last-seen timestamp, sorted by device ID",
"produces": [
"application/json"
],
"tags": [
"QueryLogs"
],
"summary": "Get profile query log devices",
"parameters": [
{
"type": "string",
"description": "Profile ID",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/model.QueryLogDevice"
}
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.ErrResponse"
}
},
"429": {
"description": "Too Many Requests",
"schema": {
"$ref": "#/definitions/api.ErrResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrResponse"
}
}
}
}
},
"/api/v1/profiles/{id}/logs/download": {
"get": {
"security": [
Expand Down Expand Up @@ -3817,6 +3872,17 @@ const docTemplate = `{
}
}
},
"model.QueryLogDevice": {
"type": "object",
"properties": {
"device_id": {
"type": "string"
},
"last_seen": {
"type": "string"
}
}
},
"model.RebindingProtection": {
"type": "object",
"properties": {
Expand Down
66 changes: 66 additions & 0 deletions api/docs/swagger.json
Original file line number Diff line number Diff line change
Expand Up @@ -1883,6 +1883,61 @@
}
}
},
"/api/v1/profiles/{id}/logs/devices": {
"get": {
"security": [
{
"ApiKeyAuth": []
}
],
"description": "List distinct device IDs seen in the profile's query logs (current retention window), each with its last-seen timestamp, sorted by device ID",
"produces": [
"application/json"
],
"tags": [
"QueryLogs"
],
"summary": "Get profile query log devices",
"parameters": [
{
"type": "string",
"description": "Profile ID",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/model.QueryLogDevice"
}
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/api.ErrResponse"
}
},
"429": {
"description": "Too Many Requests",
"schema": {
"$ref": "#/definitions/api.ErrResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/api.ErrResponse"
}
}
}
}
},
"/api/v1/profiles/{id}/logs/download": {
"get": {
"security": [
Expand Down Expand Up @@ -3809,6 +3864,17 @@
}
}
},
"model.QueryLogDevice": {
"type": "object",
"properties": {
"device_id": {
"type": "string"
},
"last_seen": {
"type": "string"
}
}
},
"model.RebindingProtection": {
"type": "object",
"properties": {
Expand Down
43 changes: 43 additions & 0 deletions api/docs/swagger.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,13 @@ definitions:
timestamp:
type: string
type: object
model.QueryLogDevice:
properties:
device_id:
type: string
last_seen:
type: string
type: object
model.RebindingProtection:
properties:
enabled:
Expand Down Expand Up @@ -2608,6 +2615,42 @@ paths:
summary: Get profile query logs
tags:
- QueryLogs
/api/v1/profiles/{id}/logs/devices:
get:
description: List distinct device IDs seen in the profile's query logs (current
retention window), each with its last-seen timestamp, sorted by device ID
parameters:
- description: Profile ID
in: path
name: id
required: true
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
items:
$ref: '#/definitions/model.QueryLogDevice'
type: array
"404":
description: Not Found
schema:
$ref: '#/definitions/api.ErrResponse'
"429":
description: Too Many Requests
schema:
$ref: '#/definitions/api.ErrResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/api.ErrResponse'
security:
- ApiKeyAuth: []
summary: Get profile query log devices
tags:
- QueryLogs
/api/v1/profiles/{id}/logs/download:
get:
description: Download profile query logs
Expand Down
1 change: 1 addition & 0 deletions api/internal/middleware/subscription_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ func TestIsLimitedAccessAllowed(t *testing.T) {
{"GET", "/api/v1/profiles", true},
{"GET", "/api/v1/profiles/abc123", true},
{"GET", "/api/v1/profiles/abc123/logs", true},
{"GET", "/api/v1/profiles/abc123/logs/devices", true},
{"GET", "/api/v1/profiles/abc123/logs/download", true},
{"DELETE", "/api/v1/profiles/abc123/logs", true},
{"GET", "/api/v1/profiles/abc123/statistics", true},
Expand Down
Loading
Loading