feat: add Webhook Deliveries query type

.changeset/wild-webhooks-report.md

@@ -0,0 +1,5 @@
+---
+'grafana-github-datasource': minor
+---
+
+🚀 Add Webhook Deliveries query type for monitoring webhook delivery success rates and failures

docs/sources/query-editor.md

@@ -50,6 +50,7 @@ Select a query type from the **Query Type** drop-down in the query editor:
 - [**Stargazers**](#stargazers): Get a list of users who have starred a repository, including the ability to plot a total count over time.
 - [**Tags**](#tags): List created tags for a repository.
 - [**Vulnerabilities**](#vulnerabilities): Query security vulnerabilities detected in a repository.
+- [**Webhook deliveries**](#webhook-deliveries): List deliveries of an organization or repository webhook, including the response status code.
 - [**Workflows**](#workflows): List GitHub Actions workflows defined in a repository.
 - [**Workflow runs**](#workflow-runs): List runs for a specific workflow, including status, conclusion, and timing information.
 - [**Workflow usage**](#workflow-usage): Retrieve usage statistics for a workflow, such as run counts and durations.
@@ -875,6 +876,58 @@ Show all security advisories for the `grafana/grafana` repository:
 | severity | Severity level: `LOW`, `MODERATE`, `HIGH`, or `CRITICAL` |
 | state | State of the vulnerability alert: `OPEN`, `FIXED`, or `DISMISSED` |
 
+### Webhook deliveries
+
+List deliveries of an organization or repository webhook, one row per delivery. Useful for tracking webhook reliability: plot the success rate of an automation webhook over time, or alert when failures pass a threshold.
+
+{{< admonition type="note" >}}
+GitHub only keeps webhook deliveries for the past 3 days, so a longer dashboard time range returns only the deliveries that are left. This query also requires a token with the `admin:org_hook` scope for organization webhooks, or `admin:repo_hook` for repository webhooks.
+{{< /admonition >}}
+
+To find the ID of a webhook, run `gh api /orgs/<org>/hooks` for an organization webhook, or `gh api /repos/<owner>/<repository>/hooks` for a repository webhook.
+
+#### Query options
+
+| Name | Description | Required |
+|------|-------------|----------|
+| Owner | Organization the webhook belongs to, or GitHub user or organization that owns the repository | Yes |
+| Repository | Name of the repository. Leave it empty to query an organization webhook | No |
+| Hook ID | Numeric ID of the webhook | Yes |
+| Event | Only return deliveries triggered by this event (for example, `pull_request`) | No |
+| Status | Filter deliveries by outcome: `All`, `Success` (a 2xx response), or `Failure` (any other response, including deliveries that never got one) | No |
+
+##### Sample queries
+
+Show all deliveries of an organization webhook:
+
+- Owner: `grafana`
+- Hook ID: `12345678`
+
+Show failed `pull_request` deliveries of a repository webhook:
+
+- Owner: `grafana`
+- Repository: `grafana`
+- Hook ID: `12345678`
+- Event: `pull_request`
+- Status: `Failure`
+
+#### Response
+
+| Name | Description |
+|------|-------------|
+| delivered_at | When the delivery was attempted: YYYY-MM-DD HH:MM:SS |
+| id | Unique identifier for the delivery |
+| guid | Identifier shared by a delivery and all of its redeliveries |
+| event | Event that triggered the delivery (for example, `pull_request`) |
+| action | Action of the event, if applicable (for example, `opened`) |
+| status | Description of the response (for example, `OK`) |
+| status_code | HTTP status code the webhook endpoint responded with |
+| duration | How long the delivery took, in seconds |
+| redelivery | Whether the delivery is a redelivery of an earlier one |
+| repository_id | ID of the repository the event came from. The delivery does not include the repository name |
+| installation_id | ID of the GitHub App installation, if the webhook belongs to one |
+| success | Whether the endpoint responded with a 2xx status code |
+
 ### Workflows
 
 List GitHub Actions workflows defined in a repository.

pkg/github/client/client.go

@@ -225,6 +225,24 @@ func (client *Client) ListDeployments(ctx context.Context, owner, repo string, o
 	return deployments, resp, err
 }
 
+// ListOrgHookDeliveries sends a request to the GitHub rest API to list the deliveries of an organization webhook.
+func (client *Client) ListOrgHookDeliveries(ctx context.Context, org string, hookID int64, opts *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	deliveries, resp, err := client.restClient.Organizations.ListHookDeliveries(ctx, org, hookID, opts)
+	if err != nil {
+		return nil, nil, addErrorSourceToError(err, resp)
+	}
+	return deliveries, resp, err
+}
+
+// ListRepoHookDeliveries sends a request to the GitHub rest API to list the deliveries of a repository webhook.
+func (client *Client) ListRepoHookDeliveries(ctx context.Context, owner, repo string, hookID int64, opts *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	deliveries, resp, err := client.restClient.Repositories.ListHookDeliveries(ctx, owner, repo, hookID, opts)
+	if err != nil {
+		return nil, nil, addErrorSourceToError(err, resp)
+	}
+	return deliveries, resp, err
+}
+
 // GetCommitFiles returns the list of files changed in a specific commit.
 // Note: the GitHub API returns at most 300 files for a single commit.
 func (client *Client) GetCommitFiles(ctx context.Context, owner, repo, sha string, opts *googlegithub.ListOptions) ([]*googlegithub.CommitFile, *googlegithub.Response, error) {

pkg/github/codescanning_test.go

@@ -297,3 +297,11 @@ func equalInts(a, b []int) bool {
 	}
 	return true
 }
+
+func (m *mockClient) ListOrgHookDeliveries(_ context.Context, _ string, _ int64, _ *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	return nil, nil, nil
+}
+
+func (m *mockClient) ListRepoHookDeliveries(_ context.Context, _, _ string, _ int64, _ *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	return nil, nil, nil
+}

pkg/github/commit_files_test.go

@@ -256,3 +256,11 @@ func TestCommitFilesFrames(t *testing.T) {
 
 	testutil.CheckGoldenFramer(t, "commit_files", files)
 }
+
+func (m *commitFilesMockClient) ListOrgHookDeliveries(_ context.Context, _ string, _ int64, _ *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	return nil, nil, nil
+}
+
+func (m *commitFilesMockClient) ListRepoHookDeliveries(_ context.Context, _, _ string, _ int64, _ *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	return nil, nil, nil
+}

pkg/github/commits_test.go

@@ -251,3 +251,11 @@ func TestCommitsWithFilesDataframe(t *testing.T) {
 
 	testutil.CheckGoldenFramer(t, "commits_with_files", cwf)
 }
+
+func (m *commitsWithFilesMockClient) ListOrgHookDeliveries(_ context.Context, _ string, _ int64, _ *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	return nil, nil, nil
+}
+
+func (m *commitsWithFilesMockClient) ListRepoHookDeliveries(_ context.Context, _, _ string, _ int64, _ *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	return nil, nil, nil
+}

pkg/github/datasource.go

@@ -91,6 +91,18 @@ func (d *Datasource) HandleBranchesQuery(ctx context.Context, query *models.Bran
 	return GetAllBranches(ctx, d.client, opt)
 }
 
+// HandleWebhookDeliveriesQuery is the query handler for listing the deliveries of a GitHub webhook
+func (d *Datasource) HandleWebhookDeliveriesQuery(ctx context.Context, query *models.WebhookDeliveriesQuery, req backend.DataQuery) (dfutil.Framer, error) {
+	opt := models.ListWebhookDeliveriesOptions{
+		Repository: query.Repository,
+		Owner:      query.Owner,
+		HookID:     query.Options.HookID,
+		Event:      query.Options.Event,
+		Status:     query.Options.Status,
+	}
+	return GetWebhookDeliveries(ctx, d.client, opt, req.TimeRange)
+}
+
 // HandleReleasesQuery is the query handler for listing GitHub Releases
 func (d *Datasource) HandleReleasesQuery(ctx context.Context, query *models.ReleasesQuery, req backend.DataQuery) (dfutil.Framer, error) {
 	opt := models.ListReleasesOptions{

pkg/github/deployments_test.go

@@ -337,3 +337,11 @@ func TestDeploymentsWrapperFrames(t *testing.T) {
 		}
 	}
 }
+
+func (m *mockDeploymentsClient) ListOrgHookDeliveries(_ context.Context, _ string, _ int64, _ *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	return nil, nil, nil
+}
+
+func (m *mockDeploymentsClient) ListRepoHookDeliveries(_ context.Context, _, _ string, _ int64, _ *googlegithub.ListCursorOptions) ([]*googlegithub.HookDelivery, *googlegithub.Response, error) {
+	return nil, nil, nil
+}

pkg/github/query_handler.go

@@ -69,6 +69,7 @@ func GetQueryHandlers(s *QueryHandler) *datasource.QueryTypeMux {
 	register(models.QueryTypeOrganizations, s.HandleOrganizations)
 	register(models.QueryTypeCommitFiles, s.HandleCommitFiles)
 	register(models.QueryTypePullRequestFiles, s.HandlePullRequestFiles)
+	register(models.QueryTypeWebhookDeliveries, s.HandleWebhookDeliveries)
 
 	return mux
 }

pkg/github/webhook_deliveries.go

@@ -0,0 +1,196 @@
+package github
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	googlegithub "github.com/google/go-github/v84/github"
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+
+	"github.com/grafana/github-datasource/pkg/models"
+)
+
+const (
+	// webhookDeliveriesPerPage is the maximum page size the deliveries endpoint accepts
+	webhookDeliveriesPerPage = 100
+
+	// maxWebhookDeliveryPages caps how many requests a single query can send. The endpoint has no time filter,
+	// so a busy webhook would otherwise page through everything GitHub kept and exhaust the rate limit.
+	maxWebhookDeliveryPages = 50
+)
+
+// WebhookDeliveries is a list of deliveries of a GitHub webhook
+type WebhookDeliveries struct {
+	deliveries []*googlegithub.HookDelivery
+
+	// truncated is set when the page cap was reached before the start of the time range, which means the
+	// deliveries only cover part of it
+	truncated bool
+
+	// retentionLimited is set when GitHub ran out of deliveries before the start of the time range, which
+	// means the deliveries only cover the part of it that is still within GitHub's retention window
+	retentionLimited bool
+}
+
+// Frames converts the list of webhook deliveries to a Grafana DataFrame
+func (w WebhookDeliveries) Frames() data.Frames {
+	frame := data.NewFrame(
+		"webhook_deliveries",
+		data.NewField("delivered_at", nil, []*time.Time{}),
+		data.NewField("id", nil, []*int64{}),
+		data.NewField("guid", nil, []*string{}),
+		data.NewField("event", nil, []*string{}),
+		data.NewField("action", nil, []*string{}),
+		data.NewField("status", nil, []*string{}),
+		data.NewField("status_code", nil, []*int64{}),
+		data.NewField("duration", nil, []*float64{}),
+		data.NewField("redelivery", nil, []*bool{}),
+		data.NewField("repository_id", nil, []*int64{}),
+		data.NewField("installation_id", nil, []*int64{}),
+		data.NewField("success", nil, []bool{}),
+	)
+
+	for _, delivery := range w.deliveries {
+		var statusCode *int64
+		if delivery.StatusCode != nil {
+			statusCode = googlegithub.Ptr(int64(*delivery.StatusCode))
+		}
+
+		frame.AppendRow(
+			delivery.DeliveredAt.GetTime(),
+			delivery.ID,
+			delivery.GUID,
+			delivery.Event,
+			delivery.Action,
+			delivery.Status,
+			statusCode,
+			delivery.Duration,
+			delivery.Redelivery,
+			delivery.RepositoryID,
+			delivery.InstallationID,
+			isSuccessfulDelivery(delivery),
+		)
+	}
+
+	frame.Meta = &data.FrameMeta{PreferredVisualization: data.VisTypeTable}
+	switch {
+	case w.truncated:
+		frame.Meta.Notices = []data.Notice{{
+			Severity: data.NoticeSeverityWarning,
+			Text: fmt.Sprintf(
+				"Only the %d most recent deliveries are shown because they do not cover the whole time range. Narrow the time range for complete results.",
+				maxWebhookDeliveryPages*webhookDeliveriesPerPage,
+			),
+		}}
+	case w.retentionLimited:
+		frame.Meta.Notices = []data.Notice{{
+			Severity: data.NoticeSeverityWarning,
+			Text:     "The deliveries do not cover the whole time range. GitHub only keeps webhook deliveries for the past 3 days.",
+		}}
+	}
+
+	return data.Frames{frame}
+}
+
+// GetWebhookDeliveries gets the deliveries of an organization or repository webhook within the time range.
+// GitHub only keeps deliveries for the past 3 days, so longer time ranges return what is left of them.
+func GetWebhookDeliveries(ctx context.Context, client models.Client, opts models.ListWebhookDeliveriesOptions, timeRange backend.TimeRange) (WebhookDeliveries, error) {
+	if opts.Owner == "" || strings.TrimSpace(opts.HookID) == "" {
+		return WebhookDeliveries{}, nil
+	}
+
+	hookID, err := strconv.ParseInt(strings.TrimSpace(opts.HookID), 10, 64)
+	if err != nil {
+		return WebhookDeliveries{}, backend.DownstreamError(fmt.Errorf("hook id must be a number, got %q", opts.HookID))
+	}
+
+	var (
+		result   = WebhookDeliveries{}
+		listOpts = &googlegithub.ListCursorOptions{PerPage: webhookDeliveriesPerPage}
+
+		// oldestDelivered is the delivery time of the last delivery seen, which is the oldest one because the
+		// endpoint returns them from the most recent one
+		oldestDelivered *time.Time
+	)
+
+	// The endpoint has no time filter, so paging always starts at the most recent delivery. A time range that
+	// ends in the past therefore still pages through everything that happened since.
+
+	for page := 0; page < maxWebhookDeliveryPages; page++ {
+		var (
+			deliveries []*googlegithub.HookDelivery
+			resp       *googlegithub.Response
+			err        error
+		)
+
+		if opts.Repository == "" {
+			deliveries, resp, err = client.ListOrgHookDeliveries(ctx, opts.Owner, hookID, listOpts)
+		} else {
+			deliveries, resp, err = client.ListRepoHookDeliveries(ctx, opts.Owner, opts.Repository, hookID, listOpts)
+		}
+		if err != nil {
+			return WebhookDeliveries{}, fmt.Errorf("listing webhook deliveries: opts=%+v: %w", opts, err)
+		}
+
+		// Deliveries are returned from the most recent one, so the first delivery older than the time range
+		// means every remaining one is older too.
+		reachedStartOfTimeRange := false
+		for _, delivery := range deliveries {
+			deliveredAt := delivery.DeliveredAt.GetTime()
+			if deliveredAt != nil && deliveredAt.Before(timeRange.From) {
+				reachedStartOfTimeRange = true
+				break
+			}
+			oldestDelivered = deliveredAt
+
+			if matchesWebhookDeliveryFilters(delivery, opts, timeRange) {
+				result.deliveries = append(result.deliveries, delivery)
+			}
+		}
+
+		if reachedStartOfTimeRange {
+			return result, nil
+		}
+
+		if resp == nil || resp.Cursor == "" {
+			// GitHub has nothing older to return, so the deliveries only reach as far back as its retention
+			// window does
+			result.retentionLimited = oldestDelivered != nil && oldestDelivered.After(timeRange.From)
+			return result, nil
+		}
+		listOpts.Cursor = resp.Cursor
+	}
+
+	result.truncated = true
+	return result, nil
+}
+
+func matchesWebhookDeliveryFilters(delivery *googlegithub.HookDelivery, opts models.ListWebhookDeliveriesOptions, timeRange backend.TimeRange) bool {
+	if deliveredAt := delivery.DeliveredAt.GetTime(); deliveredAt != nil && deliveredAt.After(timeRange.To) {
+		return false
+	}
+
+	if opts.Event != "" && delivery.GetEvent() != opts.Event {
+		return false
+	}
+
+	switch opts.Status {
+	case models.WebhookDeliveryStatusSuccess:
+		return isSuccessfulDelivery(delivery)
+	case models.WebhookDeliveryStatusFailure:
+		return !isSuccessfulDelivery(delivery)
+	}
+
+	return true
+}
+
+// isSuccessfulDelivery reports whether the webhook endpoint answered with a 2xx status code. Deliveries that
+// never got a response, for example because of a timeout, have no status code and count as failures.
+func isSuccessfulDelivery(delivery *googlegithub.HookDelivery) bool {
+	statusCode := delivery.GetStatusCode()
+	return statusCode >= 200 && statusCode < 300
+}

pkg/github/webhook_deliveries_handler.go

@@ -0,0 +1,24 @@
+package github
+
+import (
+	"context"
+
+	"github.com/grafana/github-datasource/pkg/dfutil"
+	"github.com/grafana/github-datasource/pkg/models"
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+)
+
+func (s *QueryHandler) handleWebhookDeliveriesQuery(ctx context.Context, q backend.DataQuery) backend.DataResponse {
+	query := &models.WebhookDeliveriesQuery{}
+	if err := UnmarshalQuery(q.JSON, query); err != nil {
+		return *err
+	}
+	return dfutil.FrameResponseWithError(s.Datasource.HandleWebhookDeliveriesQuery(ctx, query, q))
+}
+
+// HandleWebhookDeliveries handles the plugin query for github webhook deliveries
+func (s *QueryHandler) HandleWebhookDeliveries(ctx context.Context, req *backend.QueryDataRequest) (*backend.QueryDataResponse, error) {
+	return &backend.QueryDataResponse{
+		Responses: processQueries(ctx, req, s.handleWebhookDeliveriesQuery),
+	}, nil
+}