events: verify, normalize, append-only store with provider-id dedup
Signature verification (gitea hex HMAC-SHA256 over the raw body via constant-time hmac.Equal; redmine/discourse constant-time shared-secret headers) with one generic ErrUnverified so rejects give attackers no oracle. Tolerant normalization of the known Redmine/Discourse/Gitea payload variants into one Event record (canonical subject ids, actor, title, repo, sha256 payload digest) plus the DESIGN action mapping (dispatch_turn / respond_turn / pipeline_step / ignore). Store: JSONL under state dir, 0600, dedup keyed on provider event id (delivery header, payload-digest fallback), index rebuilt at startup so replays across restarts still dedup; torn tail lines skipped, not fatal.
This commit is contained in:
@@ -0,0 +1,276 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Provider delivery/event id headers used for dedup before falling back to
|
||||
// the payload digest.
|
||||
const (
|
||||
giteaDeliveryHeader = "X-Gitea-Delivery"
|
||||
discourseEventIDHeader = "X-Discourse-Event-Id"
|
||||
redmineDeliveryFallback = "X-Redmine-Delivery" // some plugins send it
|
||||
)
|
||||
|
||||
// Normalize maps a verified provider payload to the internal Event record.
|
||||
// It is deliberately tolerant: providers (and their plugin ecosystems)
|
||||
// disagree on payload shapes, so extraction tries the known variants and
|
||||
// leaves anything missing empty rather than rejecting. Unparseable bodies
|
||||
// are the only hard error.
|
||||
func Normalize(source string, header http.Header, body []byte, now time.Time) (Event, error) {
|
||||
var root map[string]any
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.UseNumber()
|
||||
if err := dec.Decode(&root); err != nil {
|
||||
return Event{}, fmt.Errorf("payload is not a JSON object: %w", err)
|
||||
}
|
||||
|
||||
sum := sha256.Sum256(body)
|
||||
ev := Event{
|
||||
Source: source,
|
||||
PayloadDigest: hex.EncodeToString(sum[:]),
|
||||
ReceivedAt: now.UTC(),
|
||||
}
|
||||
|
||||
switch source {
|
||||
case SourceRedmine:
|
||||
normalizeRedmine(&ev, header, root)
|
||||
case SourceDiscourse:
|
||||
normalizeDiscourse(&ev, header, root)
|
||||
case SourceGitea:
|
||||
normalizeGitea(&ev, header, root)
|
||||
default:
|
||||
return Event{}, fmt.Errorf("unknown event source %q", source)
|
||||
}
|
||||
|
||||
// Dedup identity: the provider's event id, else the payload digest
|
||||
// (identical replays dedup; different payloads never collide).
|
||||
if ev.ProviderID == "" {
|
||||
ev.ProviderID = ev.PayloadDigest
|
||||
}
|
||||
if ev.Actor == "" {
|
||||
ev.Actor = "unknown"
|
||||
}
|
||||
if ev.Kind == "" {
|
||||
ev.Kind = "unknown"
|
||||
}
|
||||
ev.Action = MapAction(ev.Source, ev.Kind)
|
||||
return ev, nil
|
||||
}
|
||||
|
||||
// MapAction implements the DESIGN trigger -> action mapping:
|
||||
// redmine issue update/note -> dispatch_turn; discourse post reply ->
|
||||
// respond_turn; gitea PR approved/merged -> pipeline_step; else ignore.
|
||||
func MapAction(source, kind string) string {
|
||||
switch source {
|
||||
case SourceRedmine:
|
||||
if kind == "issue_updated" || kind == "issue_edited" ||
|
||||
strings.Contains(kind, "note") || strings.Contains(kind, "journal") {
|
||||
return ActionDispatchTurn
|
||||
}
|
||||
case SourceDiscourse:
|
||||
if kind == "post_created" || kind == "post_edited" || kind == "post" {
|
||||
return ActionRespondTurn
|
||||
}
|
||||
case SourceGitea:
|
||||
if kind == "pr_approved" || kind == "pr_merged" {
|
||||
return ActionPipelineStep
|
||||
}
|
||||
}
|
||||
return ActionIgnore
|
||||
}
|
||||
|
||||
// normalizeRedmine handles the redmine_webhooks shape
|
||||
// {"event_name": "...", "payload": {"issue": {...}}} and flatter plugin
|
||||
// payloads that put "issue" (and the actor) at the top level.
|
||||
func normalizeRedmine(ev *Event, header http.Header, root map[string]any) {
|
||||
p := obj(root, "payload")
|
||||
issue := obj(p, "issue")
|
||||
if issue == nil {
|
||||
issue = obj(root, "issue")
|
||||
}
|
||||
ev.Kind = firstNonEmpty(
|
||||
str(root, "event_name"),
|
||||
str(p, "event_name"),
|
||||
header.Get("X-Redmine-Event"),
|
||||
)
|
||||
ev.Actor = firstNonEmpty(
|
||||
person(p, "user", "author"),
|
||||
person(root, "user", "author"),
|
||||
person(issue, "user", "author"),
|
||||
)
|
||||
if issue != nil {
|
||||
if ev.Kind == "" {
|
||||
ev.Kind = "issue_updated"
|
||||
}
|
||||
if id := numStr(issue, "id"); id != "" {
|
||||
ev.SubjectID = "redmine:issue:" + id
|
||||
}
|
||||
ev.Subject = str(issue, "subject")
|
||||
}
|
||||
ev.ProviderID = firstNonEmpty(header.Get(redmineDeliveryFallback))
|
||||
}
|
||||
|
||||
// normalizeDiscourse handles Discourse webhook bodies
|
||||
// {"post": {"id":.., "topic_id":.., "username":.., ...}}; the event type
|
||||
// rides in the X-Discourse-Event header (e.g. post_created).
|
||||
func normalizeDiscourse(ev *Event, header http.Header, root map[string]any) {
|
||||
ev.Kind = firstNonEmpty(
|
||||
header.Get("X-Discourse-Event"),
|
||||
header.Get("X-Discourse-Event-Type"),
|
||||
str(root, "event_type"),
|
||||
)
|
||||
post := obj(root, "post")
|
||||
if post != nil {
|
||||
if ev.Kind == "" {
|
||||
ev.Kind = "post_created"
|
||||
}
|
||||
if id := numStr(post, "topic_id", "topicId"); id != "" {
|
||||
ev.SubjectID = "discourse:topic:" + id
|
||||
} else if id := numStr(root, "topic_id"); id != "" {
|
||||
ev.SubjectID = "discourse:topic:" + id
|
||||
}
|
||||
ev.Subject = str(post, "topic_title", "topic_slug")
|
||||
ev.Actor = firstNonEmpty(str(post, "username"), person(root, "user"), str(root, "username"))
|
||||
} else {
|
||||
if id := numStr(root, "topic_id"); id != "" {
|
||||
ev.SubjectID = "discourse:topic:" + id
|
||||
}
|
||||
ev.Actor = firstNonEmpty(person(root, "user"), str(root, "username"))
|
||||
}
|
||||
ev.ProviderID = firstNonEmpty(header.Get(discourseEventIDHeader), str(root, "id"))
|
||||
}
|
||||
|
||||
// normalizeGitea handles Gitea webhook bodies. PR events carry
|
||||
// {"action": ..., "number": N, "pull_request": {...}, "repository": {...},
|
||||
// "sender": {...}}; issue events mirror that with "issue". kind vocabulary:
|
||||
// pr_approved / pr_merged / pr_opened / ... and issue_*.
|
||||
func normalizeGitea(ev *Event, header http.Header, root map[string]any) {
|
||||
ev.ProviderID = header.Get(giteaDeliveryHeader)
|
||||
action := str(root, "action")
|
||||
if repo := obj(root, "repository"); repo != nil {
|
||||
ev.Repo = str(repo, "full_name")
|
||||
}
|
||||
ev.Actor = person(root, "sender")
|
||||
|
||||
pr := obj(root, "pull_request")
|
||||
issue := obj(root, "issue")
|
||||
switch {
|
||||
case pr != nil:
|
||||
kind := action
|
||||
if action == "closed" && truthy(pr, "merged") {
|
||||
kind = "merged"
|
||||
}
|
||||
if kind == "" {
|
||||
kind = "unknown"
|
||||
}
|
||||
ev.Kind = "pr_" + kind
|
||||
if n := firstNonEmpty(numStr(root, "number"), numStr(pr, "number")); n != "" {
|
||||
ev.SubjectID = giteaRef(ev.Repo, "pr", n)
|
||||
}
|
||||
ev.Subject = str(pr, "title")
|
||||
case issue != nil:
|
||||
if action == "" {
|
||||
action = "updated"
|
||||
}
|
||||
ev.Kind = "issue_" + action
|
||||
if n := firstNonEmpty(numStr(root, "number"), numStr(issue, "number"), numStr(issue, "id")); n != "" {
|
||||
ev.SubjectID = giteaRef(ev.Repo, "issue", n)
|
||||
}
|
||||
ev.Subject = str(issue, "title")
|
||||
default:
|
||||
ev.Kind = firstNonEmpty(
|
||||
header.Get("X-Gitea-Event-Type"),
|
||||
header.Get("X-Gitea-Event"),
|
||||
action,
|
||||
"unknown",
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func giteaRef(repo, kind, n string) string {
|
||||
if repo == "" {
|
||||
return "gitea:" + kind + ":" + n
|
||||
}
|
||||
return "gitea:" + kind + ":" + repo + "#" + n
|
||||
}
|
||||
|
||||
// --- tolerant JSON extraction helpers -------------------------------------
|
||||
|
||||
func obj(m map[string]any, key string) map[string]any {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
if v, ok := m[key].(map[string]any); ok {
|
||||
return v
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// str returns the first non-empty string among the keys.
|
||||
func str(m map[string]any, keys ...string) string {
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
for _, k := range keys {
|
||||
if v, ok := m[k].(string); ok && v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// numStr returns the first present numeric (or string) key rendered as a
|
||||
// decimal string; ids must survive without float mangling.
|
||||
func numStr(m map[string]any, keys ...string) string {
|
||||
if m == nil {
|
||||
return ""
|
||||
}
|
||||
for _, k := range keys {
|
||||
switch v := m[k].(type) {
|
||||
case json.Number:
|
||||
return v.String()
|
||||
case string:
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
case float64:
|
||||
return fmt.Sprintf("%d", int64(v))
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// person extracts an actor from nested user-ish objects: login, name,
|
||||
// username, full_name in order.
|
||||
func person(m map[string]any, keys ...string) string {
|
||||
for _, k := range keys {
|
||||
if o := obj(m, k); o != nil {
|
||||
if s := str(o, "login", "username", "name", "full_name"); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func truthy(m map[string]any, key string) bool {
|
||||
v, ok := m[key].(bool)
|
||||
return ok && v
|
||||
}
|
||||
|
||||
func firstNonEmpty(vals ...string) string {
|
||||
for _, v := range vals {
|
||||
if v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user