From efb213a5d2cad1522db58de0e139d70f987a65d3 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 30 Jun 2026 11:33:30 +0800 Subject: [PATCH 1/6] feat: add Slack SR guard --- tools/slack_sr_monitor/.env.example | 9 + tools/slack_sr_monitor/.gitignore | 3 +- tools/slack_sr_monitor/Dockerfile | 2 +- tools/slack_sr_monitor/README.md | 45 +- tools/slack_sr_monitor/docker-compose.yml | 16 + tools/slack_sr_monitor/main.go | 528 ++----------------- tools/slack_sr_monitor/sr_guard.go | 348 ++++++++++++ tools/slack_sr_monitor/sr_monitor.go | 616 ++++++++++++++++++++++ 8 files changed, 1083 insertions(+), 484 deletions(-) create mode 100644 tools/slack_sr_monitor/sr_guard.go create mode 100644 tools/slack_sr_monitor/sr_monitor.go diff --git a/tools/slack_sr_monitor/.env.example b/tools/slack_sr_monitor/.env.example index 46ce6add..efc9786f 100644 --- a/tools/slack_sr_monitor/.env.example +++ b/tools/slack_sr_monitor/.env.example @@ -3,6 +3,15 @@ # The Slack Webhook URL for sending notifications SLACK_WEBHOOK=your_slack_webhook_url_here +# Optional per-tool Slack webhook URLs. +# If not set, each tool falls back to SLACK_WEBHOOK. +# SLACK_SR_MONITOR_WEBHOOK=your_monitor_slack_webhook_url_here +# SLACK_SR_GUARD_WEBHOOK=your_guard_slack_webhook_url_here + # The Tron node API endpoint # Default: https://api.trongrid.io TRON_NODE=https://api.trongrid.io + +# Optional comma-separated fallback Tron node API endpoints. +# If set, TRON_NODES takes precedence over TRON_NODE. +# TRON_NODES=https://api.trongrid.io,http://127.0.0.1:8090 diff --git a/tools/slack_sr_monitor/.gitignore b/tools/slack_sr_monitor/.gitignore index d775c44f..a38f37f7 100644 --- a/tools/slack_sr_monitor/.gitignore +++ b/tools/slack_sr_monitor/.gitignore @@ -1,3 +1,4 @@ -slack_sr_monitor +/slack_sr_monitor +/slack_sr_guard logs/ .env diff --git a/tools/slack_sr_monitor/Dockerfile b/tools/slack_sr_monitor/Dockerfile index 1b8d4668..7c3b5dbd 100644 --- a/tools/slack_sr_monitor/Dockerfile +++ b/tools/slack_sr_monitor/Dockerfile @@ -11,7 +11,7 @@ RUN go mod download COPY . . # Build the application -RUN CGO_ENABLED=0 GOOS=linux go build -o slack_sr_monitor main.go +RUN CGO_ENABLED=0 GOOS=linux go build -o slack_sr_monitor . # Final stage FROM alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc diff --git a/tools/slack_sr_monitor/README.md b/tools/slack_sr_monitor/README.md index 70aaef31..e68b53eb 100644 --- a/tools/slack_sr_monitor/README.md +++ b/tools/slack_sr_monitor/README.md @@ -2,8 +2,10 @@ The Slack SR Monitor tool is designed to monitor TRON Super Representatives (SRs) and notify a Slack channel after every maintenance period. It automatically tracks vote changes and detects replacements in the top 27 SR positions, providing a clear and formatted report. -### Build and Run the monitor -To run the monitor tool, you can choose between native Go execution or Docker deployment. +This directory also includes `slack_sr_guard`, a lightweight early-warning tool that polls the live SR list every minute and alerts Slack as soon as the current Top 27 membership changes. + +### Build and Run +To run either tool, choose native Go execution or Docker deployment. #### Native Go Execution Make sure you have Go 1.25+ installed. @@ -12,24 +14,48 @@ Make sure you have Go 1.25+ installed. cd tools/slack_sr_monitor # install dependencies go mod tidy -# run the tool -go run main.go +# run both tools by default +go run . + +# run only the maintenance-period monitor +go run . monitor + +# run only the early-warning guard +go run . guard ``` #### Docker Deployment We provide a Docker-based deployment for easier management in production environments. ```shell -# build and start the container -docker-compose up -d --build +# build and start both tools +docker compose up -d --build + +# build and start only one tool +docker compose up -d --build slack-sr-monitor +docker compose up -d --build slack-sr-guard + # check logs docker logs -f slack-sr-monitor +docker logs -f slack-sr-guard ``` ### Configuration All configurations are managed via environment variables or a `.env` file in the project root. Please refer to [.env.example](./.env.example) as an example. - `SLACK_WEBHOOK`: The Slack Incoming Webhook URL used to send notifications. +- `SLACK_SR_MONITOR_WEBHOOK`: Optional Slack webhook URL for `slack_sr_monitor`. Falls back to `SLACK_WEBHOOK` if unset. +- `SLACK_SR_GUARD_WEBHOOK`: Optional Slack webhook URL for `slack_sr_guard`. Falls back to `SLACK_WEBHOOK` if unset. - `TRON_NODE`: The TRON node HTTP API endpoint (e.g., `http://https://api.trongrid.io`). Default is Trongrid. +- `TRON_NODES`: Optional comma-separated TRON node HTTP API endpoints. If set, the tools try each node in order for every TRON API request and only return an error after all nodes fail. This takes precedence over `TRON_NODE`. + +### State Persistence + +Both tools persist lightweight JSON state under `logs/`, which is already mounted by Docker Compose: + +- `logs/sr_monitor_state.json`: previous vote counts and Top 27 snapshot for maintenance-period reports. +- `logs/sr_guard_state.json`: previous Top 27 snapshot and SR name cache for one-minute guard checks. + +If these files are deleted, the next run starts with a fresh baseline and recreates them after the first successful fetch. ### Key Features @@ -43,7 +69,7 @@ Instead of a fixed interval, the tool queries `/wallet/getnextmaintenancetime` t The tool uses Go routines to fetch `account_name` for all 28 witnesses in parallel from the `/wallet/getaccount` interface, significantly reducing the collection time. #### Vote Change Tracking -The tool maintains an in-memory snapshot of the previous period's votes. It calculates the `Change` for each SR: +The tool persists a snapshot of the previous period's votes. It calculates the `Change` for each SR: ```text *1. Poloniex* Current: `3,228,089,488` Change: `+89,488` @@ -58,6 +84,11 @@ SR Replacement Detected: ``` If no changes occur, it displays `Top 27 SRs remain unchanged.` +#### Top 27 Early Warning Guard +The `slack_sr_guard` process calls `/wallet/getpaginatednowwitnesslist` every minute with `limit=28`, keeps a persisted Top 27 snapshot, and sends a Slack alert only when an SR enters or leaves the Top 27 before the next maintenance period. + +The alert includes the entered SRs, left SRs, current 27th/28th vote gap, and a Top 28 vote table. + ### Notifications This monitor only support java-tron node v4.8.1+, because of the API it used. diff --git a/tools/slack_sr_monitor/docker-compose.yml b/tools/slack_sr_monitor/docker-compose.yml index ff727028..410f373e 100644 --- a/tools/slack_sr_monitor/docker-compose.yml +++ b/tools/slack_sr_monitor/docker-compose.yml @@ -3,8 +3,24 @@ services: build: . container_name: slack-sr-monitor restart: unless-stopped + command: ["./slack_sr_monitor", "monitor"] environment: - SLACK_WEBHOOK=${SLACK_WEBHOOK} + - SLACK_SR_MONITOR_WEBHOOK=${SLACK_SR_MONITOR_WEBHOOK:-} + - TRON_NODES=${TRON_NODES:-} + - TRON_NODE=${TRON_NODE:-https://api.trongrid.io} + volumes: + - ./logs:/home/monitor/logs + + slack-sr-guard: + build: . + container_name: slack-sr-guard + restart: unless-stopped + command: ["./slack_sr_monitor", "guard"] + environment: + - SLACK_WEBHOOK=${SLACK_WEBHOOK} + - SLACK_SR_GUARD_WEBHOOK=${SLACK_SR_GUARD_WEBHOOK:-} + - TRON_NODES=${TRON_NODES:-} - TRON_NODE=${TRON_NODE:-https://api.trongrid.io} volumes: - ./logs:/home/monitor/logs diff --git a/tools/slack_sr_monitor/main.go b/tools/slack_sr_monitor/main.go index 089b4b85..57a247f0 100644 --- a/tools/slack_sr_monitor/main.go +++ b/tools/slack_sr_monitor/main.go @@ -1,508 +1,86 @@ package main import ( - "bytes" - "encoding/hex" - "encoding/json" "fmt" "io" "log" - "net/http" "os" "path/filepath" "strings" "sync" - "time" "github.com/joho/godotenv" ) -const ( - DefaultTronNode = "https://api.trongrid.io" -) - -// Witness represents the structure of a witness returned by the Tron API -type Witness struct { - Address string `json:"address"` - VoteCount int64 `json:"voteCount"` - PubKey string `json:"pubKey"` - URL string `json:"url"` - TotalProduced int64 `json:"totalProduced"` - TotalMissed int64 `json:"totalMissed"` - LatestBlock int64 `json:"latestBlockNum"` - IsJobs bool `json:"isJobs"` - DisplayName string `json:"-"` -} - -func getAccountName(nodeURL string, address string) string { - url := fmt.Sprintf("%s/wallet/getaccount", nodeURL) - payload := map[string]interface{}{ - "address": address, - "visible": true, - } - jsonPayload, err := json.Marshal(payload) - if err != nil { - log.Printf("Error: failed to marshal account name payload for %s: %v", address, err) - return "" - } - - client := &http.Client{Timeout: 5 * time.Second} - resp, err := client.Post(url, "application/json", bytes.NewBuffer(jsonPayload)) - if err != nil { - log.Printf("Error: failed to request account name for %s: %v", address, err) - return "" - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - log.Printf("Error: failed to read account response body for %s: %v", address, err) - return "" - } - - var data map[string]interface{} - if err := json.Unmarshal(body, &data); err != nil { - log.Printf("Error: failed to unmarshal account JSON for %s: %v", address, err) - return "" - } - - if val, ok := data["account_name"]; ok { - if str, ok := val.(string); ok { - return str - } - return fmt.Sprintf("%v", val) - } - - return "" -} - -// WitnessListResponse is the wrapper for the witness list API response -type WitnessListResponse struct { - Witnesses []Witness `json:"witnesses"` -} - -// NextMaintenanceResponse is the wrapper for the maintenance time API response -type NextMaintenanceResponse struct { - Num int64 `json:"num"` -} - -func getNextMaintenanceTime(nodeURL string) (time.Time, error) { - url := fmt.Sprintf("%s/wallet/getnextmaintenancetime", nodeURL) - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Post(url, "application/json", nil) - if err != nil { - return time.Time{}, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return time.Time{}, fmt.Errorf("status code %d", resp.StatusCode) - } - - var result NextMaintenanceResponse - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return time.Time{}, err - } - - // Result is in milliseconds - return time.Unix(result.Num/1000, (result.Num%1000)*1000000), nil -} - -// NowBlockResponse captures the timestamp of the node's latest block. -type NowBlockResponse struct { - BlockHeader struct { - RawData struct { - Number int64 `json:"number"` - Timestamp int64 `json:"timestamp"` - } `json:"raw_data"` - } `json:"block_header"` -} - -// A fresh TRON node should always be within a few block intervals (block time is 3s). -// Anything beyond this threshold means the node is still catching up to the chain head. -const maxBlockAge = 30 * time.Second - -func getLatestBlockTime(nodeURL string) (time.Time, error) { - url := fmt.Sprintf("%s/wallet/getnowblock", nodeURL) - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Post(url, "application/json", nil) - if err != nil { - return time.Time{}, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return time.Time{}, fmt.Errorf("status code %d", resp.StatusCode) - } - - var result NowBlockResponse - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return time.Time{}, err - } - - ts := result.BlockHeader.RawData.Timestamp - return time.Unix(ts/1000, (ts%1000)*1000000), nil -} - -func getWitnessList(nodeURL string) ([]Witness, error) { - url := fmt.Sprintf("%s/wallet/getpaginatednowwitnesslist", nodeURL) - - payload := []byte(`{"offset": 0, "limit": 28, "visible": true}`) - - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Post(url, "application/json", bytes.NewBuffer(payload)) - if err != nil { - return nil, fmt.Errorf("failed to call Tron API: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("tron API returned status %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %v", err) - } - - var result WitnessListResponse - if err := json.Unmarshal(body, &result); err != nil { - return nil, fmt.Errorf("failed to unmarshal JSON: %v (body: %s)", err, string(body)) - } - - // For each witness, try to get the account name in parallel - log.Printf("Fetching account names for %d witnesses in parallel...\n", len(result.Witnesses)) - var wg sync.WaitGroup - for i := range result.Witnesses { - wg.Add(1) - go func(idx int) { - defer wg.Done() - w := &result.Witnesses[idx] - accName := getAccountName(nodeURL, w.Address) - if accName != "" { - if decoded, err := hex.DecodeString(accName); err == nil { - w.DisplayName = string(decoded) - } else { - w.DisplayName = accName - } - } - if w.DisplayName == "" { - w.DisplayName = w.URL - } - }(i) - } - wg.Wait() - - return result.Witnesses, nil -} - -func formatComma(n int64) string { - in := fmt.Sprintf("%d", n) - var out strings.Builder - if n < 0 { - out.WriteByte('-') - in = in[1:] - } - l := len(in) - for i, c := range in { - if i > 0 && (l-i)%3 == 0 { - out.WriteByte(',') - } - out.WriteRune(c) - } - return out.String() -} - -func sendToSlack(webhookURL string, witnesses []Witness, prevVotes map[string]int64, prevSRs []string) error { - var summary strings.Builder - summary.WriteString("*TRON SR Status Update (Maintenance Period)*") - summary.WriteString(fmt.Sprintf(" Time: %s\n", time.Now().UTC().Format(time.RFC1123))) - - // 1. Calculate SR changes for the summary - if len(prevSRs) > 0 { - currentTop27 := make(map[string]bool) - for i := 0; i < 27 && i < len(witnesses); i++ { - currentTop27[witnesses[i].Address] = true - } - - prevTop27 := make(map[string]bool) - for _, addr := range prevSRs { - prevTop27[addr] = true - } - - var entered []string - var left []string - - // Who enter - for i := 0; i < 27 && i < len(witnesses); i++ { - w := witnesses[i] - if !prevTop27[w.Address] { - name := w.DisplayName - if name == "" { - name = w.Address - } - entered = append(entered, name) - } - } - - // Who left - for _, addr := range prevSRs { - if !currentTop27[addr] { - name := addr - for _, w := range witnesses { - if w.Address == addr { - name = w.DisplayName - if name == "" { - name = w.Address - } - break - } - } - left = append(left, name) - } - } - - // Build rank map for previous SRs - prevRankMap := make(map[string]int) - for i, addr := range prevSRs { - prevRankMap[addr] = i + 1 // Rank is 1-based - } - - // Detect rank changes within top 27 - var rankChanges []string - for i := 0; i < 27 && i < len(witnesses); i++ { - w := witnesses[i] - if prevRank, ok := prevRankMap[w.Address]; ok { - change := prevRank - (i + 1) // Positive = rank up, negative = rank down - if change != 0 { - direction := "↑" - if change < 0 { - direction = "↓" - change = -change - } - name := w.DisplayName - if name == "" { - name = w.Address - } - rankChanges = append(rankChanges, - fmt.Sprintf("%s: %d → %d (%s%d)", name, prevRank, i+1, direction, change)) - } - } - } - - // Output all changes (entering, leaving, and rank changes) in one block - if len(entered) > 0 || len(left) > 0 || len(rankChanges) > 0 { - summary.WriteString("*SR Changes Detected:*\n") - if len(entered) > 0 { - summary.WriteString(fmt.Sprintf(">:inbox_tray: *Entered:* %s\n", strings.Join(entered, ", "))) - } - if len(left) > 0 { - summary.WriteString(fmt.Sprintf(">:outbox_tray: *Left:* %s\n", strings.Join(left, ", "))) - } - if len(rankChanges) > 0 { - summary.WriteString(">*Rank Changes:*\n") - for _, rc := range rankChanges { - summary.WriteString(fmt.Sprintf("> `%s`\n", rc)) - } - } - } else { - summary.WriteString("*Top 27 SRs remain unchanged.*\n") - } - } else { - summary.WriteString("*First check, initializing SR list*\n") - } - - // Calculate gap between 27th and 28th SR (always show this if we have enough witnesses) - if len(witnesses) >= 28 { - v27 := witnesses[26].VoteCount - v28 := witnesses[27].VoteCount - gap := v27 - v28 - summary.WriteString(fmt.Sprintf("*Gap (27 vs 28):* `%s` votes", formatComma(gap))) - } - - // 2. Build detailed list for attachment - var details strings.Builder - details.WriteString("```\n") - for i, w := range witnesses { - name := w.DisplayName - if name == "" { - name = w.Address - } - - prev := prevVotes[w.Address] - diff := w.VoteCount - prev - - diffStr := formatComma(diff) - if diff >= 0 { - diffStr = "+" + diffStr - } - if prev == 0 { - diffStr = "-" - } - - details.WriteString(fmt.Sprintf("%2d. %-25s Current: %15s Change: %10s\n", - i+1, name, formatComma(w.VoteCount), diffStr)) - } - details.WriteString("```") - - // 3. Construct Payload with Attachments - payload := map[string]interface{}{ - "text": summary.String(), - "attachments": []map[string]interface{}{ - { - "text": details.String(), - "color": "#36a64f", - }, - }, - } - - jsonPayload, err := json.Marshal(payload) - if err != nil { - return fmt.Errorf("failed to marshal slack payload: %v", err) - } - - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) - if err != nil { - return fmt.Errorf("failed to send to slack: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) - } - - return nil -} - -func updateLastStatus(witnesses []Witness, lastVotes map[string]int64) []string { - var top27 []string - for i, w := range witnesses { - lastVotes[w.Address] = w.VoteCount - if i < 27 { - top27 = append(top27, w.Address) - } - } - return top27 -} - -// The final path segment of a Slack webhook URL is the secret token; mask it before logging. -func maskWebhook(url string) string { - idx := strings.LastIndex(url, "/") - if idx < 0 || idx == len(url)-1 { - return "***" - } - return url[:idx+1] + "***" -} - -func main() { - // Ensure logs directory exists +func setupLogging() { logDir := "logs" if _, err := os.Stat(logDir); os.IsNotExist(err) { _ = os.Mkdir(logDir, 0755) } - // Setup logging to both file and stdout - logPath := filepath.Join(logDir, "sr_monitor.log") + logPath := filepath.Join(logDir, "slack_sr_tools.log") logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) if err != nil { fmt.Printf("Error opening log file: %v, falling back to stdout only\n", err) } else { - mw := io.MultiWriter(os.Stdout, logFile) - log.SetOutput(mw) + log.SetOutput(io.MultiWriter(os.Stdout, logFile)) } log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) +} + +func main() { + setupLogging() - // Load .env file if err := godotenv.Load(); err != nil { log.Println("Warning: No .env file found, using system environment variables") } - tronNode := os.Getenv("TRON_NODE") - if tronNode == "" { - tronNode = DefaultTronNode - } - - slackWebhook := os.Getenv("SLACK_WEBHOOK") - if slackWebhook == "" { - log.Println("Error: SLACK_WEBHOOK environment variable is not set") - log.Println("Usage: SLACK_WEBHOOK=https://hooks.slack.com/... [TRON_NODE=...] go run main.go") + tronNodes := parseTronNodes() + mode := "both" + if len(os.Args) > 1 { + mode = strings.ToLower(os.Args[1]) + } + + switch mode { + case "monitor", "sr-monitor", "slack-sr-monitor": + slackWebhook := getSlackWebhook("SLACK_SR_MONITOR_WEBHOOK") + if slackWebhook == "" { + log.Println("Error: SLACK_SR_MONITOR_WEBHOOK or SLACK_WEBHOOK environment variable is not set") + log.Println("Usage: SLACK_SR_MONITOR_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run . monitor") + os.Exit(1) + } + runSRMonitor(tronNodes, slackWebhook) + case "guard", "sr-guard", "slack-sr-guard": + slackWebhook := getSlackWebhook("SLACK_SR_GUARD_WEBHOOK") + if slackWebhook == "" { + log.Println("Error: SLACK_SR_GUARD_WEBHOOK or SLACK_WEBHOOK environment variable is not set") + log.Println("Usage: SLACK_SR_GUARD_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run . guard") + os.Exit(1) + } + runSRGuard(tronNodes, slackWebhook) + case "both", "": + monitorWebhook := getSlackWebhook("SLACK_SR_MONITOR_WEBHOOK") + guardWebhook := getSlackWebhook("SLACK_SR_GUARD_WEBHOOK") + if monitorWebhook == "" || guardWebhook == "" { + log.Println("Error: SLACK_SR_MONITOR_WEBHOOK/SLACK_SR_GUARD_WEBHOOK or SLACK_WEBHOOK environment variable is not set") + log.Println("Usage: SLACK_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run .") + os.Exit(1) + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + runSRMonitor(tronNodes, monitorWebhook) + }() + go func() { + defer wg.Done() + runSRGuard(tronNodes, guardWebhook) + }() + wg.Wait() + default: + log.Printf("Error: unknown mode %q", mode) + log.Println("Usage: go run . [monitor|guard|both]") os.Exit(1) } - - log.Printf("Starting SR monitor.\nNode: %s\nSlack Webhook: %s\n", tronNode, maskWebhook(slackWebhook)) - - // Map to track votes: Address -> VoteCount - lastVotes := make(map[string]int64) - var lastTop27 []string - - // Initial check - log.Println("Performing initial check...") - witnesses, err := getWitnessList(tronNode) - if err != nil { - log.Printf("Initial check failed: %v\n", err) - } else { - log.Printf("Successfully fetched %d witnesses. Sending to Slack...\n", len(witnesses)) - if err := sendToSlack(slackWebhook, witnesses, lastVotes, lastTop27); err != nil { - log.Printf("Failed to send initial update to Slack: %v\n", err) - } else { - lastTop27 = updateLastStatus(witnesses, lastVotes) - } - } - - log.Println("Monitoring for maintenance periods via getnextmaintenancetime...") - - for { - nextTime, err := getNextMaintenanceTime(tronNode) - if err != nil { - log.Printf("Error fetching next maintenance time: %v, retrying in 1 minute...\n", err) - time.Sleep(1 * time.Minute) - continue - } - - // A node that is still catching up will report a past maintenance time and would - // otherwise cause this loop to send duplicate Slack messages every 2 minutes. - blockTime, err := getLatestBlockTime(tronNode) - if err != nil { - log.Printf("Error fetching latest block: %v, retrying in 1 minute...\n", err) - time.Sleep(1 * time.Minute) - continue - } - if age := time.Since(blockTime); age > maxBlockAge { - log.Printf("Node is %s behind chain head (latest block at %s), waiting 2 minutes...\n", - age.Truncate(time.Second), blockTime.UTC().Format(time.RFC3339)) - time.Sleep(2 * time.Minute) - continue - } - - // Calculate trigger time: next maintenance time + 1 minute - triggerTime := nextTime.Add(1 * time.Minute) - now := time.Now().UTC() - waitDuration := triggerTime.Sub(now) - - if waitDuration > 0 { - log.Printf("Next maintenance time: %s (UTC). Waiting %v until %s...\n", - nextTime.Format(time.RFC1123), waitDuration.Truncate(time.Second), triggerTime.Format(time.RFC1123)) - time.Sleep(waitDuration) - } else { - log.Printf("Maintenance time %s has already passed. Checking now...\n", nextTime.Format(time.RFC1123)) - } - - log.Printf("[%s] Maintenance period reached (+1m). Fetching SR list...\n", time.Now().UTC().Format(time.RFC3339)) - witnesses, err := getWitnessList(tronNode) - if err != nil { - log.Printf("Error fetching witness list: %v\n", err) - } else { - if err := sendToSlack(slackWebhook, witnesses, lastVotes, lastTop27); err != nil { - log.Printf("Error sending to Slack: %v\n", err) - } else { - log.Println("Successfully sent SR list to Slack") - lastTop27 = updateLastStatus(witnesses, lastVotes) - } - } - - // Wait a bit before checking for the NEXT maintenance time to avoid double-triggering - time.Sleep(2 * time.Minute) - } } diff --git a/tools/slack_sr_monitor/sr_guard.go b/tools/slack_sr_monitor/sr_guard.go new file mode 100644 index 00000000..277c56be --- /dev/null +++ b/tools/slack_sr_monitor/sr_guard.go @@ -0,0 +1,348 @@ +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const ( + pollInterval = time.Minute + topSRLimit = 27 + fetchLimit = 28 + guardStatePath = "logs/sr_guard_state.json" + guardStateVersion = 1 +) + +type guardState struct { + Version int `json:"version"` + UpdatedAt time.Time `json:"updated_at"` + PreviousTop27 []string `json:"previous_top_27"` + NameCache map[string]string `json:"name_cache"` +} + +func getGuardWitnessListFromNodes(nodeURLs []string) ([]Witness, error) { + payload := []byte(fmt.Sprintf(`{"offset":0,"limit":%d,"visible":true}`, fetchLimit)) + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getpaginatednowwitnesslist", payload, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var result WitnessListResponse + if err := json.Unmarshal(body, &result); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal JSON: %v", nodeURL, err)) + continue + } + + return result.Witnesses, nil + } + + return nil, fmt.Errorf("all TRON nodes failed for /wallet/getpaginatednowwitnesslist: %s", strings.Join(errors, "; ")) +} + +func getGuardAccountNameFromNodes(nodeURLs []string, address string) string { + payload := map[string]interface{}{ + "address": address, + "visible": true, + } + jsonPayload, err := json.Marshal(payload) + if err != nil { + log.Printf("failed to marshal account name payload for %s: %v", address, err) + return "" + } + + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getaccount", jsonPayload, 5*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + name, ok := parseAccountName(address, body) + if !ok { + errors = append(errors, fmt.Sprintf("%s: invalid account JSON", nodeURL)) + continue + } + + return name + } + + log.Printf("failed to request account name for %s: %s", address, strings.Join(errors, "; ")) + return "" +} + +func parseAccountName(address string, body []byte) (string, bool) { + var data map[string]interface{} + if err := json.Unmarshal(body, &data); err != nil { + log.Printf("failed to unmarshal account JSON for %s: %v", address, err) + return "", false + } + + val, ok := data["account_name"] + if !ok { + return "", true + } + + str, ok := val.(string) + if !ok { + str = fmt.Sprintf("%v", val) + } + decoded, err := hex.DecodeString(str) + if err != nil { + return str, true + } + return string(decoded), true +} + +func fillDisplayNames(nodeURLs []string, witnesses []Witness, nameCache map[string]string) { + var wg sync.WaitGroup + var mu sync.Mutex + + for i := range witnesses { + if name, ok := nameCache[witnesses[i].Address]; ok { + witnesses[i].DisplayName = name + continue + } + + wg.Add(1) + go func(idx int) { + defer wg.Done() + + witness := witnesses[idx] + name := getGuardAccountNameFromNodes(nodeURLs, witness.Address) + if name == "" { + name = witness.URL + } + if name == "" { + name = witness.Address + } + + mu.Lock() + nameCache[witness.Address] = name + witnesses[idx].DisplayName = name + mu.Unlock() + }(i) + } + + wg.Wait() +} + +func topAddresses(witnesses []Witness) []string { + limit := topSRLimit + if len(witnesses) < limit { + limit = len(witnesses) + } + + addresses := make([]string, 0, limit) + for i := 0; i < limit; i++ { + addresses = append(addresses, witnesses[i].Address) + } + return addresses +} + +func findTopSRChanges(previous []string, current []Witness) ([]Witness, []string) { + previousSet := make(map[string]bool, len(previous)) + for _, address := range previous { + previousSet[address] = true + } + + currentSet := make(map[string]bool, topSRLimit) + var entered []Witness + for i := 0; i < topSRLimit && i < len(current); i++ { + witness := current[i] + currentSet[witness.Address] = true + if !previousSet[witness.Address] { + entered = append(entered, witness) + } + } + + var left []string + for _, address := range previous { + if !currentSet[address] { + left = append(left, address) + } + } + + return entered, left +} + +func witnessName(w Witness) string { + if w.DisplayName != "" { + return w.DisplayName + } + if w.URL != "" { + return w.URL + } + return w.Address +} + +func sendAlert(webhookURL string, witnesses []Witness, entered []Witness, left []string, nameCache map[string]string) error { + var text strings.Builder + text.WriteString("*:warning: TRON Top 27 SR Change Detected*\n") + text.WriteString(fmt.Sprintf("Time: %s\n", time.Now().UTC().Format(time.RFC1123))) + + if len(entered) > 0 { + var names []string + for _, witness := range entered { + names = append(names, fmt.Sprintf("%s (`%s` votes)", witnessName(witness), formatComma(witness.VoteCount))) + } + text.WriteString(fmt.Sprintf(">:inbox_tray: *Entered Top 27:* %s\n", strings.Join(names, ", "))) + } + + if len(left) > 0 { + var names []string + for _, address := range left { + name := nameCache[address] + if name == "" { + name = address + } + names = append(names, name) + } + text.WriteString(fmt.Sprintf(">:outbox_tray: *Left Top 27:* %s\n", strings.Join(names, ", "))) + } + + if len(witnesses) >= fetchLimit { + gap := witnesses[26].VoteCount - witnesses[27].VoteCount + text.WriteString(fmt.Sprintf("*Gap (27 vs 28):* `%s` votes\n", formatComma(gap))) + text.WriteString(fmt.Sprintf("*27:* %s `%s` votes\n", witnessName(witnesses[26]), formatComma(witnesses[26].VoteCount))) + text.WriteString(fmt.Sprintf("*28:* %s `%s` votes\n", witnessName(witnesses[27]), formatComma(witnesses[27].VoteCount))) + } + + var details strings.Builder + details.WriteString("```\n") + for i, witness := range witnesses { + details.WriteString(fmt.Sprintf("%2d. %-25s %15s\n", i+1, witnessName(witness), formatComma(witness.VoteCount))) + } + details.WriteString("```") + + payload := map[string]interface{}{ + "text": text.String(), + "attachments": []map[string]interface{}{ + { + "text": details.String(), + "color": "#e01e5a", + }, + }, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send to slack: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func loadGuardState() guardState { + state := guardState{ + Version: guardStateVersion, + NameCache: make(map[string]string), + } + + body, err := os.ReadFile(guardStatePath) + if err != nil { + if !os.IsNotExist(err) { + log.Printf("failed to read guard state: %v", err) + } + return state + } + + if err := json.Unmarshal(body, &state); err != nil { + log.Printf("failed to parse guard state: %v", err) + return guardState{Version: guardStateVersion, NameCache: make(map[string]string)} + } + if state.NameCache == nil { + state.NameCache = make(map[string]string) + } + log.Printf("loaded guard state: top27=%d names=%d updated_at=%s", len(state.PreviousTop27), len(state.NameCache), state.UpdatedAt.UTC().Format(time.RFC3339)) + return state +} + +func saveGuardState(previousTop27 []string, nameCache map[string]string) { + state := guardState{ + Version: guardStateVersion, + UpdatedAt: time.Now().UTC(), + PreviousTop27: previousTop27, + NameCache: nameCache, + } + body, err := json.MarshalIndent(state, "", " ") + if err != nil { + log.Printf("failed to marshal guard state: %v", err) + return + } + if err := os.MkdirAll(filepath.Dir(guardStatePath), 0755); err != nil { + log.Printf("failed to create guard state directory: %v", err) + return + } + if err := os.WriteFile(guardStatePath, body, 0644); err != nil { + log.Printf("failed to write guard state: %v", err) + } +} + +func runSRGuard(tronNodes []string, slackWebhook string) { + log.Printf("Starting SR guard. Nodes: %s Slack Webhook: %s Poll interval: %s", strings.Join(tronNodes, ", "), maskWebhook(slackWebhook), pollInterval) + + state := loadGuardState() + nameCache := state.NameCache + previousTop27 := state.PreviousTop27 + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + for { + witnesses, err := getGuardWitnessListFromNodes(tronNodes) + if err != nil { + log.Printf("failed to fetch witness list: %v", err) + } else { + fillDisplayNames(tronNodes, witnesses, nameCache) + + if len(previousTop27) == 0 { + previousTop27 = topAddresses(witnesses) + saveGuardState(previousTop27, nameCache) + log.Printf("initialized Top 27 snapshot with %d witnesses", len(previousTop27)) + } else { + entered, left := findTopSRChanges(previousTop27, witnesses) + if len(entered) > 0 || len(left) > 0 { + log.Printf("Top 27 change detected: entered=%d left=%d", len(entered), len(left)) + if err := sendAlert(slackWebhook, witnesses, entered, left, nameCache); err != nil { + log.Printf("failed to send Slack alert: %v", err) + } else { + previousTop27 = topAddresses(witnesses) + saveGuardState(previousTop27, nameCache) + log.Println("sent Slack alert and updated Top 27 snapshot") + } + } else { + previousTop27 = topAddresses(witnesses) + saveGuardState(previousTop27, nameCache) + log.Println("Top 27 unchanged") + } + } + } + + <-ticker.C + } +} diff --git a/tools/slack_sr_monitor/sr_monitor.go b/tools/slack_sr_monitor/sr_monitor.go new file mode 100644 index 00000000..f481f53c --- /dev/null +++ b/tools/slack_sr_monitor/sr_monitor.go @@ -0,0 +1,616 @@ +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const ( + DefaultTronNode = "https://api.trongrid.io" + monitorStatePath = "logs/sr_monitor_state.json" + monitorStateVersion = 1 +) + +type monitorState struct { + Version int `json:"version"` + UpdatedAt time.Time `json:"updated_at"` + LastVotes map[string]int64 `json:"last_votes"` + LastTop27 []string `json:"last_top_27"` +} + +// Witness represents the structure of a witness returned by the Tron API +type Witness struct { + Address string `json:"address"` + VoteCount int64 `json:"voteCount"` + PubKey string `json:"pubKey"` + URL string `json:"url"` + TotalProduced int64 `json:"totalProduced"` + TotalMissed int64 `json:"totalMissed"` + LatestBlock int64 `json:"latestBlockNum"` + IsJobs bool `json:"isJobs"` + DisplayName string `json:"-"` +} + +func getAccountNameFromNodes(nodeURLs []string, address string) string { + payload := map[string]interface{}{ + "address": address, + "visible": true, + } + jsonPayload, err := json.Marshal(payload) + if err != nil { + log.Printf("Error: failed to marshal account name payload for %s: %v", address, err) + return "" + } + + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getaccount", jsonPayload, 5*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var data map[string]interface{} + if err := json.Unmarshal(body, &data); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal account JSON: %v", nodeURL, err)) + continue + } + + if val, ok := data["account_name"]; ok { + if str, ok := val.(string); ok { + return str + } + return fmt.Sprintf("%v", val) + } + + return "" + } + + log.Printf("Error: failed to request account name for %s: %s", address, strings.Join(errors, "; ")) + return "" +} + +func parseTronNodes() []string { + raw := os.Getenv("TRON_NODES") + if raw == "" { + raw = os.Getenv("TRON_NODE") + } + if raw == "" { + raw = DefaultTronNode + } + + var nodes []string + for _, node := range strings.Split(raw, ",") { + node = strings.TrimSpace(node) + if node != "" { + nodes = append(nodes, strings.TrimRight(node, "/")) + } + } + if len(nodes) == 0 { + return []string{DefaultTronNode} + } + return nodes +} + +func getSlackWebhook(specificEnv string) string { + if webhook := os.Getenv(specificEnv); webhook != "" { + return webhook + } + return os.Getenv("SLACK_WEBHOOK") +} + +func postToTron(nodeURLs []string, path string, payload []byte, timeout time.Duration) ([]byte, string, error) { + client := &http.Client{Timeout: timeout} + var errors []string + + for _, nodeURL := range nodeURLs { + resp, err := client.Post(nodeURL+path, "application/json", bytes.NewBuffer(payload)) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", nodeURL, err)) + continue + } + + body, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + if readErr != nil { + errors = append(errors, fmt.Sprintf("%s: failed to read response body: %v", nodeURL, readErr)) + continue + } + if resp.StatusCode != http.StatusOK { + errors = append(errors, fmt.Sprintf("%s: status %d: %s", nodeURL, resp.StatusCode, string(body))) + continue + } + + return body, nodeURL, nil + } + + return nil, "", fmt.Errorf("all TRON nodes failed for %s: %s", path, strings.Join(errors, "; ")) +} + +// WitnessListResponse is the wrapper for the witness list API response +type WitnessListResponse struct { + Witnesses []Witness `json:"witnesses"` +} + +// NextMaintenanceResponse is the wrapper for the maintenance time API response +type NextMaintenanceResponse struct { + Num int64 `json:"num"` +} + +func getNextMaintenanceTimeFromNodes(nodeURLs []string) (time.Time, error) { + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getnextmaintenancetime", nil, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var result NextMaintenanceResponse + if err := json.Unmarshal(body, &result); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal next maintenance JSON: %v", nodeURL, err)) + continue + } + + return time.Unix(result.Num/1000, (result.Num%1000)*1000000), nil + } + + return time.Time{}, fmt.Errorf("all TRON nodes failed for /wallet/getnextmaintenancetime: %s", strings.Join(errors, "; ")) +} + +// NowBlockResponse captures the timestamp of the node's latest block. +type NowBlockResponse struct { + BlockHeader struct { + RawData struct { + Number int64 `json:"number"` + Timestamp int64 `json:"timestamp"` + } `json:"raw_data"` + } `json:"block_header"` +} + +// A fresh TRON node should always be within a few block intervals (block time is 3s). +// Anything beyond this threshold means the node is still catching up to the chain head. +const maxBlockAge = 30 * time.Second + +func getLatestBlockTimeFromNodes(nodeURLs []string) (time.Time, error) { + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getnowblock", nil, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var result NowBlockResponse + if err := json.Unmarshal(body, &result); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal now block JSON: %v", nodeURL, err)) + continue + } + + ts := result.BlockHeader.RawData.Timestamp + return time.Unix(ts/1000, (ts%1000)*1000000), nil + } + + return time.Time{}, fmt.Errorf("all TRON nodes failed for /wallet/getnowblock: %s", strings.Join(errors, "; ")) +} + +func getWitnessListFromNodes(nodeURLs []string) ([]Witness, error) { + payload := []byte(`{"offset": 0, "limit": 28, "visible": true}`) + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getpaginatednowwitnesslist", payload, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + witnesses, err := parseWitnessList(nodeURLs, body) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", nodeURL, err)) + continue + } + + return witnesses, nil + } + + return nil, fmt.Errorf("all TRON nodes failed for /wallet/getpaginatednowwitnesslist: %s", strings.Join(errors, "; ")) +} + +func parseWitnessList(nodeURLs []string, body []byte) ([]Witness, error) { + var result WitnessListResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON: %v (body: %s)", err, string(body)) + } + + // For each witness, try to get the account name in parallel + log.Printf("Fetching account names for %d witnesses in parallel...\n", len(result.Witnesses)) + var wg sync.WaitGroup + for i := range result.Witnesses { + wg.Add(1) + go func(idx int) { + defer wg.Done() + w := &result.Witnesses[idx] + accName := getAccountNameFromNodes(nodeURLs, w.Address) + if accName != "" { + if decoded, err := hex.DecodeString(accName); err == nil { + w.DisplayName = string(decoded) + } else { + w.DisplayName = accName + } + } + if w.DisplayName == "" { + w.DisplayName = w.URL + } + }(i) + } + wg.Wait() + + return result.Witnesses, nil +} + +func formatComma(n int64) string { + in := fmt.Sprintf("%d", n) + var out strings.Builder + if n < 0 { + out.WriteByte('-') + in = in[1:] + } + l := len(in) + for i, c := range in { + if i > 0 && (l-i)%3 == 0 { + out.WriteByte(',') + } + out.WriteRune(c) + } + return out.String() +} + +func sendToSlack(webhookURL string, witnesses []Witness, prevVotes map[string]int64, prevSRs []string) error { + var summary strings.Builder + summary.WriteString("*TRON SR Status Update (Maintenance Period)*") + summary.WriteString(fmt.Sprintf(" Time: %s\n", time.Now().UTC().Format(time.RFC1123))) + + // 1. Calculate SR changes for the summary + if len(prevSRs) > 0 { + currentTop27 := make(map[string]bool) + for i := 0; i < 27 && i < len(witnesses); i++ { + currentTop27[witnesses[i].Address] = true + } + + prevTop27 := make(map[string]bool) + for _, addr := range prevSRs { + prevTop27[addr] = true + } + + var entered []string + var left []string + + // Who enter + for i := 0; i < 27 && i < len(witnesses); i++ { + w := witnesses[i] + if !prevTop27[w.Address] { + name := w.DisplayName + if name == "" { + name = w.Address + } + entered = append(entered, name) + } + } + + // Who left + for _, addr := range prevSRs { + if !currentTop27[addr] { + name := addr + for _, w := range witnesses { + if w.Address == addr { + name = w.DisplayName + if name == "" { + name = w.Address + } + break + } + } + left = append(left, name) + } + } + + // Build rank map for previous SRs + prevRankMap := make(map[string]int) + for i, addr := range prevSRs { + prevRankMap[addr] = i + 1 // Rank is 1-based + } + + // Detect rank changes within top 27 + var rankChanges []string + for i := 0; i < 27 && i < len(witnesses); i++ { + w := witnesses[i] + if prevRank, ok := prevRankMap[w.Address]; ok { + change := prevRank - (i + 1) // Positive = rank up, negative = rank down + if change != 0 { + direction := "↑" + if change < 0 { + direction = "↓" + change = -change + } + name := w.DisplayName + if name == "" { + name = w.Address + } + rankChanges = append(rankChanges, + fmt.Sprintf("%s: %d → %d (%s%d)", name, prevRank, i+1, direction, change)) + } + } + } + + // Output all changes (entering, leaving, and rank changes) in one block + if len(entered) > 0 || len(left) > 0 || len(rankChanges) > 0 { + summary.WriteString("*SR Changes Detected:*\n") + if len(entered) > 0 { + summary.WriteString(fmt.Sprintf(">:inbox_tray: *Entered:* %s\n", strings.Join(entered, ", "))) + } + if len(left) > 0 { + summary.WriteString(fmt.Sprintf(">:outbox_tray: *Left:* %s\n", strings.Join(left, ", "))) + } + if len(rankChanges) > 0 { + summary.WriteString(">*Rank Changes:*\n") + for _, rc := range rankChanges { + summary.WriteString(fmt.Sprintf("> `%s`\n", rc)) + } + } + } else { + summary.WriteString("*Top 27 SRs remain unchanged.*\n") + } + } else { + summary.WriteString("*First check, initializing SR list*\n") + } + + // Calculate gap between 27th and 28th SR (always show this if we have enough witnesses) + if len(witnesses) >= 28 { + v27 := witnesses[26].VoteCount + v28 := witnesses[27].VoteCount + gap := v27 - v28 + summary.WriteString(fmt.Sprintf("*Gap (27 vs 28):* `%s` votes", formatComma(gap))) + } + + // 2. Build detailed list for attachment + var details strings.Builder + details.WriteString("```\n") + for i, w := range witnesses { + name := w.DisplayName + if name == "" { + name = w.Address + } + + prev := prevVotes[w.Address] + diff := w.VoteCount - prev + + diffStr := formatComma(diff) + if diff >= 0 { + diffStr = "+" + diffStr + } + if prev == 0 { + diffStr = "-" + } + + details.WriteString(fmt.Sprintf("%2d. %-25s Current: %15s Change: %10s\n", + i+1, name, formatComma(w.VoteCount), diffStr)) + } + details.WriteString("```") + + // 3. Construct Payload with Attachments + payload := map[string]interface{}{ + "text": summary.String(), + "attachments": []map[string]interface{}{ + { + "text": details.String(), + "color": "#36a64f", + }, + }, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send to slack: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func sendSlackText(webhookURL string, text string) error { + payload := map[string]interface{}{ + "text": text, + } + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send to slack: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func updateLastStatus(witnesses []Witness, lastVotes map[string]int64) []string { + var top27 []string + for i, w := range witnesses { + lastVotes[w.Address] = w.VoteCount + if i < 27 { + top27 = append(top27, w.Address) + } + } + return top27 +} + +func loadMonitorState() monitorState { + state := monitorState{ + Version: monitorStateVersion, + LastVotes: make(map[string]int64), + } + + body, err := os.ReadFile(monitorStatePath) + if err != nil { + if !os.IsNotExist(err) { + log.Printf("Failed to read monitor state: %v", err) + } + return state + } + + if err := json.Unmarshal(body, &state); err != nil { + log.Printf("Failed to parse monitor state: %v", err) + return monitorState{Version: monitorStateVersion, LastVotes: make(map[string]int64)} + } + if state.LastVotes == nil { + state.LastVotes = make(map[string]int64) + } + log.Printf("Loaded monitor state: votes=%d top27=%d updated_at=%s", len(state.LastVotes), len(state.LastTop27), state.UpdatedAt.UTC().Format(time.RFC3339)) + return state +} + +func saveMonitorState(lastVotes map[string]int64, lastTop27 []string) { + state := monitorState{ + Version: monitorStateVersion, + UpdatedAt: time.Now().UTC(), + LastVotes: lastVotes, + LastTop27: lastTop27, + } + body, err := json.MarshalIndent(state, "", " ") + if err != nil { + log.Printf("Failed to marshal monitor state: %v", err) + return + } + if err := os.MkdirAll(filepath.Dir(monitorStatePath), 0755); err != nil { + log.Printf("Failed to create monitor state directory: %v", err) + return + } + if err := os.WriteFile(monitorStatePath, body, 0644); err != nil { + log.Printf("Failed to write monitor state: %v", err) + } +} + +// The final path segment of a Slack webhook URL is the secret token; mask it before logging. +func maskWebhook(url string) string { + idx := strings.LastIndex(url, "/") + if idx < 0 || idx == len(url)-1 { + return "***" + } + return url[:idx+1] + "***" +} + +func runSRMonitor(tronNodes []string, slackWebhook string) { + log.Printf("Starting SR monitor.\nNodes: %s\nSlack Webhook: %s\n", strings.Join(tronNodes, ", "), maskWebhook(slackWebhook)) + + state := loadMonitorState() + lastVotes := state.LastVotes + lastTop27 := state.LastTop27 + + if len(lastVotes) > 0 && len(lastTop27) > 0 { + text := fmt.Sprintf("TRON SR monitor started with existing state. Previous state updated at `%s`. Next report will be sent after the next maintenance period.", state.UpdatedAt.UTC().Format(time.RFC3339)) + if err := sendSlackText(slackWebhook, text); err != nil { + log.Printf("Failed to send startup log to Slack: %v\n", err) + } else { + log.Println("Sent startup log to Slack; keeping existing monitor state until next maintenance report") + } + } else { + log.Println("No complete monitor state found. Performing initial check...") + witnesses, err := getWitnessListFromNodes(tronNodes) + if err != nil { + log.Printf("Initial check failed: %v\n", err) + } else { + log.Printf("Successfully fetched %d witnesses. Sending to Slack...\n", len(witnesses)) + if err := sendToSlack(slackWebhook, witnesses, lastVotes, lastTop27); err != nil { + log.Printf("Failed to send initial update to Slack: %v\n", err) + } else { + lastTop27 = updateLastStatus(witnesses, lastVotes) + saveMonitorState(lastVotes, lastTop27) + } + } + } + + log.Println("Monitoring for maintenance periods via getnextmaintenancetime...") + + for { + nextTime, err := getNextMaintenanceTimeFromNodes(tronNodes) + if err != nil { + log.Printf("Error fetching next maintenance time: %v, retrying in 1 minute...\n", err) + time.Sleep(1 * time.Minute) + continue + } + + // A node that is still catching up will report a past maintenance time and would + // otherwise cause this loop to send duplicate Slack messages every 2 minutes. + blockTime, err := getLatestBlockTimeFromNodes(tronNodes) + if err != nil { + log.Printf("Error fetching latest block: %v, retrying in 1 minute...\n", err) + time.Sleep(1 * time.Minute) + continue + } + if age := time.Since(blockTime); age > maxBlockAge { + log.Printf("Node is %s behind chain head (latest block at %s), waiting 2 minutes...\n", + age.Truncate(time.Second), blockTime.UTC().Format(time.RFC3339)) + time.Sleep(2 * time.Minute) + continue + } + + // Calculate trigger time: next maintenance time + 1 minute + triggerTime := nextTime.Add(1 * time.Minute) + now := time.Now().UTC() + waitDuration := triggerTime.Sub(now) + + if waitDuration > 0 { + log.Printf("Next maintenance time: %s (UTC). Waiting %v until %s...\n", + nextTime.Format(time.RFC1123), waitDuration.Truncate(time.Second), triggerTime.Format(time.RFC1123)) + time.Sleep(waitDuration) + } else { + log.Printf("Maintenance time %s has already passed. Checking now...\n", nextTime.Format(time.RFC1123)) + } + + log.Printf("[%s] Maintenance period reached (+1m). Fetching SR list...\n", time.Now().UTC().Format(time.RFC3339)) + witnesses, err := getWitnessListFromNodes(tronNodes) + if err != nil { + log.Printf("Error fetching witness list: %v\n", err) + } else { + if err := sendToSlack(slackWebhook, witnesses, lastVotes, lastTop27); err != nil { + log.Printf("Error sending to Slack: %v\n", err) + } else { + log.Println("Successfully sent SR list to Slack") + lastTop27 = updateLastStatus(witnesses, lastVotes) + saveMonitorState(lastVotes, lastTop27) + } + } + + // Wait a bit before checking for the NEXT maintenance time to avoid double-triggering + time.Sleep(2 * time.Minute) + } +} From 238591fb607cabd6af1eb8397afb6b7f657f2196 Mon Sep 17 00:00:00 2001 From: warku123 Date: Tue, 30 Jun 2026 14:21:11 +0800 Subject: [PATCH 2/6] fix: retry SR monitor after maintenance --- tools/slack_sr_monitor/sr_monitor.go | 76 +++++++++++++++++++++------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/tools/slack_sr_monitor/sr_monitor.go b/tools/slack_sr_monitor/sr_monitor.go index f481f53c..0fb7ee2c 100644 --- a/tools/slack_sr_monitor/sr_monitor.go +++ b/tools/slack_sr_monitor/sr_monitor.go @@ -180,7 +180,13 @@ type NowBlockResponse struct { // A fresh TRON node should always be within a few block intervals (block time is 3s). // Anything beyond this threshold means the node is still catching up to the chain head. -const maxBlockAge = 30 * time.Second +const ( + maxBlockAge = 30 * time.Second + freshnessRetryInterval = 2 * time.Minute + freshnessRetryTimeout = 2 * time.Hour + witnessRetryInterval = 30 * time.Second + witnessRetryTimeout = 10 * time.Minute +) func getLatestBlockTimeFromNodes(nodeURLs []string) (time.Time, error) { var errors []string @@ -226,6 +232,46 @@ func getWitnessListFromNodes(nodeURLs []string) ([]Witness, error) { return nil, fmt.Errorf("all TRON nodes failed for /wallet/getpaginatednowwitnesslist: %s", strings.Join(errors, "; ")) } +func waitForFreshNode(nodeURLs []string) error { + deadline := time.Now().Add(freshnessRetryTimeout) + + for { + blockTime, err := getLatestBlockTimeFromNodes(nodeURLs) + if err != nil { + log.Printf("Error fetching latest block: %v", err) + } else if age := time.Since(blockTime); age <= maxBlockAge { + log.Printf("Node is fresh (latest block at %s, age %s)", blockTime.UTC().Format(time.RFC3339), age.Truncate(time.Second)) + return nil + } else { + log.Printf("Node is %s behind chain head (latest block at %s), waiting %s...", + age.Truncate(time.Second), blockTime.UTC().Format(time.RFC3339), freshnessRetryInterval) + } + + if time.Now().Add(freshnessRetryInterval).After(deadline) { + return fmt.Errorf("node did not become fresh within %s", freshnessRetryTimeout) + } + time.Sleep(freshnessRetryInterval) + } +} + +func getWitnessListWithRetry(nodeURLs []string) ([]Witness, error) { + deadline := time.Now().Add(witnessRetryTimeout) + + for { + witnesses, err := getWitnessListFromNodes(nodeURLs) + if err == nil { + return witnesses, nil + } + + log.Printf("Error fetching witness list: %v", err) + if time.Now().Add(witnessRetryInterval).After(deadline) { + return nil, fmt.Errorf("failed to fetch witness list within %s: %v", witnessRetryTimeout, err) + } + log.Printf("Retrying witness list in %s...", witnessRetryInterval) + time.Sleep(witnessRetryInterval) + } +} + func parseWitnessList(nodeURLs []string, body []byte) ([]Witness, error) { var result WitnessListResponse if err := json.Unmarshal(body, &result); err != nil { @@ -568,21 +614,6 @@ func runSRMonitor(tronNodes []string, slackWebhook string) { continue } - // A node that is still catching up will report a past maintenance time and would - // otherwise cause this loop to send duplicate Slack messages every 2 minutes. - blockTime, err := getLatestBlockTimeFromNodes(tronNodes) - if err != nil { - log.Printf("Error fetching latest block: %v, retrying in 1 minute...\n", err) - time.Sleep(1 * time.Minute) - continue - } - if age := time.Since(blockTime); age > maxBlockAge { - log.Printf("Node is %s behind chain head (latest block at %s), waiting 2 minutes...\n", - age.Truncate(time.Second), blockTime.UTC().Format(time.RFC3339)) - time.Sleep(2 * time.Minute) - continue - } - // Calculate trigger time: next maintenance time + 1 minute triggerTime := nextTime.Add(1 * time.Minute) now := time.Now().UTC() @@ -596,10 +627,17 @@ func runSRMonitor(tronNodes []string, slackWebhook string) { log.Printf("Maintenance time %s has already passed. Checking now...\n", nextTime.Format(time.RFC1123)) } - log.Printf("[%s] Maintenance period reached (+1m). Fetching SR list...\n", time.Now().UTC().Format(time.RFC3339)) - witnesses, err := getWitnessListFromNodes(tronNodes) + log.Printf("[%s] Maintenance period reached (+1m). Waiting for fresh node...\n", time.Now().UTC().Format(time.RFC3339)) + if err := waitForFreshNode(tronNodes); err != nil { + log.Printf("Skipping this maintenance report: %v\n", err) + time.Sleep(2 * time.Minute) + continue + } + + log.Printf("[%s] Fetching SR list...\n", time.Now().UTC().Format(time.RFC3339)) + witnesses, err := getWitnessListWithRetry(tronNodes) if err != nil { - log.Printf("Error fetching witness list: %v\n", err) + log.Printf("Skipping this maintenance report: %v\n", err) } else { if err := sendToSlack(slackWebhook, witnesses, lastVotes, lastTop27); err != nil { log.Printf("Error sending to Slack: %v\n", err) From 4bd1839603cefcd6da4d986d8b578678f1adaa57 Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 1 Jul 2026 11:27:25 +0800 Subject: [PATCH 3/6] fix: simplify SR guard alert summary --- tools/slack_sr_monitor/sr_guard.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tools/slack_sr_monitor/sr_guard.go b/tools/slack_sr_monitor/sr_guard.go index 277c56be..4b035008 100644 --- a/tools/slack_sr_monitor/sr_guard.go +++ b/tools/slack_sr_monitor/sr_guard.go @@ -214,13 +214,6 @@ func sendAlert(webhookURL string, witnesses []Witness, entered []Witness, left [ text.WriteString(fmt.Sprintf(">:outbox_tray: *Left Top 27:* %s\n", strings.Join(names, ", "))) } - if len(witnesses) >= fetchLimit { - gap := witnesses[26].VoteCount - witnesses[27].VoteCount - text.WriteString(fmt.Sprintf("*Gap (27 vs 28):* `%s` votes\n", formatComma(gap))) - text.WriteString(fmt.Sprintf("*27:* %s `%s` votes\n", witnessName(witnesses[26]), formatComma(witnesses[26].VoteCount))) - text.WriteString(fmt.Sprintf("*28:* %s `%s` votes\n", witnessName(witnesses[27]), formatComma(witnesses[27].VoteCount))) - } - var details strings.Builder details.WriteString("```\n") for i, witness := range witnesses { From 2ea164a9fb8fd0d67b5152441fad01263ef03ecd Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 1 Jul 2026 11:44:53 +0800 Subject: [PATCH 4/6] fix: resolve names for left SRs --- tools/slack_sr_monitor/sr_monitor.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/slack_sr_monitor/sr_monitor.go b/tools/slack_sr_monitor/sr_monitor.go index 0fb7ee2c..c606e8ec 100644 --- a/tools/slack_sr_monitor/sr_monitor.go +++ b/tools/slack_sr_monitor/sr_monitor.go @@ -321,7 +321,7 @@ func formatComma(n int64) string { return out.String() } -func sendToSlack(webhookURL string, witnesses []Witness, prevVotes map[string]int64, prevSRs []string) error { +func sendToSlack(webhookURL string, nodeURLs []string, witnesses []Witness, prevVotes map[string]int64, prevSRs []string) error { var summary strings.Builder summary.WriteString("*TRON SR Status Update (Maintenance Period)*") summary.WriteString(fmt.Sprintf(" Time: %s\n", time.Now().UTC().Format(time.RFC1123))) @@ -357,8 +357,10 @@ func sendToSlack(webhookURL string, witnesses []Witness, prevVotes map[string]in for _, addr := range prevSRs { if !currentTop27[addr] { name := addr + foundInCurrentList := false for _, w := range witnesses { if w.Address == addr { + foundInCurrentList = true name = w.DisplayName if name == "" { name = w.Address @@ -366,6 +368,11 @@ func sendToSlack(webhookURL string, witnesses []Witness, prevVotes map[string]in break } } + if !foundInCurrentList { + if accountName := getAccountNameFromNodes(nodeURLs, addr); accountName != "" { + name = accountName + } + } left = append(left, name) } } @@ -595,7 +602,7 @@ func runSRMonitor(tronNodes []string, slackWebhook string) { log.Printf("Initial check failed: %v\n", err) } else { log.Printf("Successfully fetched %d witnesses. Sending to Slack...\n", len(witnesses)) - if err := sendToSlack(slackWebhook, witnesses, lastVotes, lastTop27); err != nil { + if err := sendToSlack(slackWebhook, tronNodes, witnesses, lastVotes, lastTop27); err != nil { log.Printf("Failed to send initial update to Slack: %v\n", err) } else { lastTop27 = updateLastStatus(witnesses, lastVotes) @@ -639,7 +646,7 @@ func runSRMonitor(tronNodes []string, slackWebhook string) { if err != nil { log.Printf("Skipping this maintenance report: %v\n", err) } else { - if err := sendToSlack(slackWebhook, witnesses, lastVotes, lastTop27); err != nil { + if err := sendToSlack(slackWebhook, tronNodes, witnesses, lastVotes, lastTop27); err != nil { log.Printf("Error sending to Slack: %v\n", err) } else { log.Println("Successfully sent SR list to Slack") From 858b4d2b100d356bfccfd5bc807fb8e7633169e7 Mon Sep 17 00:00:00 2001 From: zack Date: Wed, 1 Jul 2026 15:31:41 +0800 Subject: [PATCH 5/6] Add phone-call escalation to slack_sr_guard Trigger a phone call via an optional PHONE_ALERT_URL webhook on the same Top 27 membership change condition that already sends the Slack alert, independent of and non-blocking to the existing Slack path. --- tools/slack_sr_monitor/.env.example | 5 ++ tools/slack_sr_monitor/README.md | 3 + tools/slack_sr_monitor/docker-compose.yml | 1 + tools/slack_sr_monitor/main.go | 5 +- tools/slack_sr_monitor/sr_guard.go | 75 ++++++++++++++++++- tools/slack_sr_monitor/sr_guard_phone_test.go | 40 ++++++++++ tools/slack_sr_monitor/sr_monitor.go | 6 ++ 7 files changed, 131 insertions(+), 4 deletions(-) create mode 100644 tools/slack_sr_monitor/sr_guard_phone_test.go diff --git a/tools/slack_sr_monitor/.env.example b/tools/slack_sr_monitor/.env.example index efc9786f..59143800 100644 --- a/tools/slack_sr_monitor/.env.example +++ b/tools/slack_sr_monitor/.env.example @@ -15,3 +15,8 @@ TRON_NODE=https://api.trongrid.io # Optional comma-separated fallback Tron node API endpoints. # If set, TRON_NODES takes precedence over TRON_NODE. # TRON_NODES=https://api.trongrid.io,http://127.0.0.1:8090 + +# Optional: have slack_sr_guard also trigger a phone/voice call alert when +# the Top 27 changes, by POSTing a short message to this webhook endpoint. +# If unset, this is skipped entirely and only the Slack alert is sent. +# PHONE_ALERT_URL=your_phone_alert_webhook_url_here diff --git a/tools/slack_sr_monitor/README.md b/tools/slack_sr_monitor/README.md index e68b53eb..395f4953 100644 --- a/tools/slack_sr_monitor/README.md +++ b/tools/slack_sr_monitor/README.md @@ -47,6 +47,7 @@ All configurations are managed via environment variables or a `.env` file in the - `SLACK_SR_GUARD_WEBHOOK`: Optional Slack webhook URL for `slack_sr_guard`. Falls back to `SLACK_WEBHOOK` if unset. - `TRON_NODE`: The TRON node HTTP API endpoint (e.g., `http://https://api.trongrid.io`). Default is Trongrid. - `TRON_NODES`: Optional comma-separated TRON node HTTP API endpoints. If set, the tools try each node in order for every TRON API request and only return an error after all nodes fail. This takes precedence over `TRON_NODE`. +- `PHONE_ALERT_URL`: Optional webhook endpoint for phone/voice call escalation. When set, `slack_sr_guard` sends a short alert message here on every Top 27 change, in addition to the Slack alert, so on-call staff can be reached even outside Slack. Skipped entirely when unset. ### State Persistence @@ -89,6 +90,8 @@ The `slack_sr_guard` process calls `/wallet/getpaginatednowwitnesslist` every mi The alert includes the entered SRs, left SRs, current 27th/28th vote gap, and a Top 28 vote table. +When `PHONE_ALERT_URL` is configured, the same Top 27 change also triggers a phone call escalation, independent of the Slack alert (a failure to reach it is logged but never blocks or fails the Slack notification). + ### Notifications This monitor only support java-tron node v4.8.1+, because of the API it used. diff --git a/tools/slack_sr_monitor/docker-compose.yml b/tools/slack_sr_monitor/docker-compose.yml index 410f373e..cae336fc 100644 --- a/tools/slack_sr_monitor/docker-compose.yml +++ b/tools/slack_sr_monitor/docker-compose.yml @@ -22,5 +22,6 @@ services: - SLACK_SR_GUARD_WEBHOOK=${SLACK_SR_GUARD_WEBHOOK:-} - TRON_NODES=${TRON_NODES:-} - TRON_NODE=${TRON_NODE:-https://api.trongrid.io} + - PHONE_ALERT_URL=${PHONE_ALERT_URL:-} volumes: - ./logs:/home/monitor/logs diff --git a/tools/slack_sr_monitor/main.go b/tools/slack_sr_monitor/main.go index 57a247f0..b8c9793d 100644 --- a/tools/slack_sr_monitor/main.go +++ b/tools/slack_sr_monitor/main.go @@ -57,7 +57,7 @@ func main() { log.Println("Usage: SLACK_SR_GUARD_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run . guard") os.Exit(1) } - runSRGuard(tronNodes, slackWebhook) + runSRGuard(tronNodes, slackWebhook, getPhoneAlertURL()) case "both", "": monitorWebhook := getSlackWebhook("SLACK_SR_MONITOR_WEBHOOK") guardWebhook := getSlackWebhook("SLACK_SR_GUARD_WEBHOOK") @@ -66,6 +66,7 @@ func main() { log.Println("Usage: SLACK_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run .") os.Exit(1) } + phoneAlertURL := getPhoneAlertURL() var wg sync.WaitGroup wg.Add(2) @@ -75,7 +76,7 @@ func main() { }() go func() { defer wg.Done() - runSRGuard(tronNodes, guardWebhook) + runSRGuard(tronNodes, guardWebhook, phoneAlertURL) }() wg.Wait() default: diff --git a/tools/slack_sr_monitor/sr_guard.go b/tools/slack_sr_monitor/sr_guard.go index 4b035008..f0eb6d7b 100644 --- a/tools/slack_sr_monitor/sr_guard.go +++ b/tools/slack_sr_monitor/sr_guard.go @@ -251,6 +251,63 @@ func sendAlert(webhookURL string, witnesses []Witness, entered []Witness, left [ return nil } +// buildPhoneAlertSummary produces a plain-text (no Slack markdown) summary, describing +// which SRs entered/left the Top 27, for the phone alert message. +func buildPhoneAlertSummary(entered []Witness, left []string, nameCache map[string]string) string { + var parts []string + + if len(entered) > 0 { + var names []string + for _, witness := range entered { + names = append(names, witnessName(witness)) + } + parts = append(parts, fmt.Sprintf("entered: %s", strings.Join(names, ", "))) + } + + if len(left) > 0 { + var names []string + for _, address := range left { + name := nameCache[address] + if name == "" { + name = address + } + names = append(names, name) + } + parts = append(parts, fmt.Sprintf("left: %s", strings.Join(names, ", "))) + } + + return "TRON Top 27 SR change detected (" + strings.Join(parts, "; ") + ")" +} + +// triggerPhoneAlert posts a short message to a webhook that triggers a phone/voice +// call alert, independent of the Slack alert sent by sendAlert. +func triggerPhoneAlert(phoneAlertURL string, entered []Witness, left []string, nameCache map[string]string) error { + summary := buildPhoneAlertSummary(entered, left, nameCache) + + payload := map[string]string{ + "message": summary, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal phone alert payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(phoneAlertURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send phone alert: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("phone alert endpoint returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + func loadGuardState() guardState { state := guardState{ Version: guardStateVersion, @@ -297,8 +354,13 @@ func saveGuardState(previousTop27 []string, nameCache map[string]string) { } } -func runSRGuard(tronNodes []string, slackWebhook string) { - log.Printf("Starting SR guard. Nodes: %s Slack Webhook: %s Poll interval: %s", strings.Join(tronNodes, ", "), maskWebhook(slackWebhook), pollInterval) +func runSRGuard(tronNodes []string, slackWebhook string, phoneAlertURL string) { + phoneLog := "disabled" + if phoneAlertURL != "" { + phoneLog = maskWebhook(phoneAlertURL) + } + log.Printf("Starting SR guard. Nodes: %s Slack Webhook: %s Phone Alert URL: %s Poll interval: %s", + strings.Join(tronNodes, ", "), maskWebhook(slackWebhook), phoneLog, pollInterval) state := loadGuardState() nameCache := state.NameCache @@ -321,6 +383,15 @@ func runSRGuard(tronNodes []string, slackWebhook string) { entered, left := findTopSRChanges(previousTop27, witnesses) if len(entered) > 0 || len(left) > 0 { log.Printf("Top 27 change detected: entered=%d left=%d", len(entered), len(left)) + + if phoneAlertURL != "" { + if err := triggerPhoneAlert(phoneAlertURL, entered, left, nameCache); err != nil { + log.Printf("failed to trigger phone alert: %v", err) + } else { + log.Println("triggered phone alert") + } + } + if err := sendAlert(slackWebhook, witnesses, entered, left, nameCache); err != nil { log.Printf("failed to send Slack alert: %v", err) } else { diff --git a/tools/slack_sr_monitor/sr_guard_phone_test.go b/tools/slack_sr_monitor/sr_guard_phone_test.go new file mode 100644 index 00000000..4982cabd --- /dev/null +++ b/tools/slack_sr_monitor/sr_guard_phone_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestTriggerPhoneAlert(t *testing.T) { + var gotBody map[string]interface{} + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("failed to decode request body: %v", err) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + entered := []Witness{{Address: "TEntered1", DisplayName: "New SR", VoteCount: 1000}} + left := []string{"TLeft1"} + nameCache := map[string]string{"TLeft1": "Old SR"} + + if err := triggerPhoneAlert(server.URL, entered, left, nameCache); err != nil { + t.Fatalf("triggerPhoneAlert returned error: %v", err) + } + + message, _ := gotBody["message"].(string) + if message == "" { + t.Fatalf("expected non-empty message field") + } + if !strings.Contains(message, "New SR") { + t.Errorf("expected message to mention entered witness 'New SR', got %q", message) + } + if !strings.Contains(message, "Old SR") { + t.Errorf("expected message to mention left witness 'Old SR', got %q", message) + } +} diff --git a/tools/slack_sr_monitor/sr_monitor.go b/tools/slack_sr_monitor/sr_monitor.go index c606e8ec..39c4ab75 100644 --- a/tools/slack_sr_monitor/sr_monitor.go +++ b/tools/slack_sr_monitor/sr_monitor.go @@ -109,6 +109,12 @@ func getSlackWebhook(specificEnv string) string { return os.Getenv("SLACK_WEBHOOK") } +// getPhoneAlertURL returns the Phone_AWS API Gateway endpoint used by slack_sr_guard to +// trigger a phone call directly. Optional; when unset, the phone alert is skipped. +func getPhoneAlertURL() string { + return os.Getenv("PHONE_ALERT_URL") +} + func postToTron(nodeURLs []string, path string, payload []byte, timeout time.Duration) ([]byte, string, error) { client := &http.Client{Timeout: timeout} var errors []string From a9f11f217c6bee6f3e1eaa2a7a9bbe8c87d56d2e Mon Sep 17 00:00:00 2001 From: warku123 Date: Wed, 1 Jul 2026 16:42:51 +0800 Subject: [PATCH 6/6] fix: trigger phone alert after Slack success --- tools/slack_sr_monitor/sr_guard.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tools/slack_sr_monitor/sr_guard.go b/tools/slack_sr_monitor/sr_guard.go index f0eb6d7b..52bea412 100644 --- a/tools/slack_sr_monitor/sr_guard.go +++ b/tools/slack_sr_monitor/sr_guard.go @@ -280,7 +280,7 @@ func buildPhoneAlertSummary(entered []Witness, left []string, nameCache map[stri } // triggerPhoneAlert posts a short message to a webhook that triggers a phone/voice -// call alert, independent of the Slack alert sent by sendAlert. +// call alert after the Slack alert has been delivered successfully. func triggerPhoneAlert(phoneAlertURL string, entered []Witness, left []string, nameCache map[string]string) error { summary := buildPhoneAlertSummary(entered, left, nameCache) @@ -384,20 +384,20 @@ func runSRGuard(tronNodes []string, slackWebhook string, phoneAlertURL string) { if len(entered) > 0 || len(left) > 0 { log.Printf("Top 27 change detected: entered=%d left=%d", len(entered), len(left)) - if phoneAlertURL != "" { - if err := triggerPhoneAlert(phoneAlertURL, entered, left, nameCache); err != nil { - log.Printf("failed to trigger phone alert: %v", err) - } else { - log.Println("triggered phone alert") - } - } - if err := sendAlert(slackWebhook, witnesses, entered, left, nameCache); err != nil { log.Printf("failed to send Slack alert: %v", err) } else { previousTop27 = topAddresses(witnesses) saveGuardState(previousTop27, nameCache) log.Println("sent Slack alert and updated Top 27 snapshot") + + if phoneAlertURL != "" { + if err := triggerPhoneAlert(phoneAlertURL, entered, left, nameCache); err != nil { + log.Printf("failed to trigger phone alert: %v", err) + } else { + log.Println("triggered phone alert") + } + } } } else { previousTop27 = topAddresses(witnesses)