diff --git a/tools/slack_sr_monitor/.env.example b/tools/slack_sr_monitor/.env.example index 46ce6ad..5914380 100644 --- a/tools/slack_sr_monitor/.env.example +++ b/tools/slack_sr_monitor/.env.example @@ -3,6 +3,20 @@ # The Slack Webhook URL for sending notifications SLACK_WEBHOOK=your_slack_webhook_url_here +# Optional per-tool Slack webhook URLs. +# If not set, each tool falls back to SLACK_WEBHOOK. +# SLACK_SR_MONITOR_WEBHOOK=your_monitor_slack_webhook_url_here +# SLACK_SR_GUARD_WEBHOOK=your_guard_slack_webhook_url_here + # The Tron node API endpoint # Default: https://api.trongrid.io TRON_NODE=https://api.trongrid.io + +# Optional comma-separated fallback Tron node API endpoints. +# If set, TRON_NODES takes precedence over TRON_NODE. +# TRON_NODES=https://api.trongrid.io,http://127.0.0.1:8090 + +# Optional: have slack_sr_guard also trigger a phone/voice call alert when +# the Top 27 changes, by POSTing a short message to this webhook endpoint. +# If unset, this is skipped entirely and only the Slack alert is sent. +# PHONE_ALERT_URL=your_phone_alert_webhook_url_here diff --git a/tools/slack_sr_monitor/.gitignore b/tools/slack_sr_monitor/.gitignore index d775c44..a38f37f 100644 --- a/tools/slack_sr_monitor/.gitignore +++ b/tools/slack_sr_monitor/.gitignore @@ -1,3 +1,4 @@ -slack_sr_monitor +/slack_sr_monitor +/slack_sr_guard logs/ .env diff --git a/tools/slack_sr_monitor/Dockerfile b/tools/slack_sr_monitor/Dockerfile index 1b8d466..7c3b5db 100644 --- a/tools/slack_sr_monitor/Dockerfile +++ b/tools/slack_sr_monitor/Dockerfile @@ -11,7 +11,7 @@ RUN go mod download COPY . . # Build the application -RUN CGO_ENABLED=0 GOOS=linux go build -o slack_sr_monitor main.go +RUN CGO_ENABLED=0 GOOS=linux go build -o slack_sr_monitor . # Final stage FROM alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc diff --git a/tools/slack_sr_monitor/README.md b/tools/slack_sr_monitor/README.md index 70aaef3..395f495 100644 --- a/tools/slack_sr_monitor/README.md +++ b/tools/slack_sr_monitor/README.md @@ -2,8 +2,10 @@ The Slack SR Monitor tool is designed to monitor TRON Super Representatives (SRs) and notify a Slack channel after every maintenance period. It automatically tracks vote changes and detects replacements in the top 27 SR positions, providing a clear and formatted report. -### Build and Run the monitor -To run the monitor tool, you can choose between native Go execution or Docker deployment. +This directory also includes `slack_sr_guard`, a lightweight early-warning tool that polls the live SR list every minute and alerts Slack as soon as the current Top 27 membership changes. + +### Build and Run +To run either tool, choose native Go execution or Docker deployment. #### Native Go Execution Make sure you have Go 1.25+ installed. @@ -12,24 +14,49 @@ Make sure you have Go 1.25+ installed. cd tools/slack_sr_monitor # install dependencies go mod tidy -# run the tool -go run main.go +# run both tools by default +go run . + +# run only the maintenance-period monitor +go run . monitor + +# run only the early-warning guard +go run . guard ``` #### Docker Deployment We provide a Docker-based deployment for easier management in production environments. ```shell -# build and start the container -docker-compose up -d --build +# build and start both tools +docker compose up -d --build + +# build and start only one tool +docker compose up -d --build slack-sr-monitor +docker compose up -d --build slack-sr-guard + # check logs docker logs -f slack-sr-monitor +docker logs -f slack-sr-guard ``` ### Configuration All configurations are managed via environment variables or a `.env` file in the project root. Please refer to [.env.example](./.env.example) as an example. - `SLACK_WEBHOOK`: The Slack Incoming Webhook URL used to send notifications. +- `SLACK_SR_MONITOR_WEBHOOK`: Optional Slack webhook URL for `slack_sr_monitor`. Falls back to `SLACK_WEBHOOK` if unset. +- `SLACK_SR_GUARD_WEBHOOK`: Optional Slack webhook URL for `slack_sr_guard`. Falls back to `SLACK_WEBHOOK` if unset. - `TRON_NODE`: The TRON node HTTP API endpoint (e.g., `http://https://api.trongrid.io`). Default is Trongrid. +- `TRON_NODES`: Optional comma-separated TRON node HTTP API endpoints. If set, the tools try each node in order for every TRON API request and only return an error after all nodes fail. This takes precedence over `TRON_NODE`. +- `PHONE_ALERT_URL`: Optional webhook endpoint for phone/voice call escalation. When set, `slack_sr_guard` sends a short alert message here on every Top 27 change, in addition to the Slack alert, so on-call staff can be reached even outside Slack. Skipped entirely when unset. + +### State Persistence + +Both tools persist lightweight JSON state under `logs/`, which is already mounted by Docker Compose: + +- `logs/sr_monitor_state.json`: previous vote counts and Top 27 snapshot for maintenance-period reports. +- `logs/sr_guard_state.json`: previous Top 27 snapshot and SR name cache for one-minute guard checks. + +If these files are deleted, the next run starts with a fresh baseline and recreates them after the first successful fetch. ### Key Features @@ -43,7 +70,7 @@ Instead of a fixed interval, the tool queries `/wallet/getnextmaintenancetime` t The tool uses Go routines to fetch `account_name` for all 28 witnesses in parallel from the `/wallet/getaccount` interface, significantly reducing the collection time. #### Vote Change Tracking -The tool maintains an in-memory snapshot of the previous period's votes. It calculates the `Change` for each SR: +The tool persists a snapshot of the previous period's votes. It calculates the `Change` for each SR: ```text *1. Poloniex* Current: `3,228,089,488` Change: `+89,488` @@ -58,6 +85,13 @@ SR Replacement Detected: ``` If no changes occur, it displays `Top 27 SRs remain unchanged.` +#### Top 27 Early Warning Guard +The `slack_sr_guard` process calls `/wallet/getpaginatednowwitnesslist` every minute with `limit=28`, keeps a persisted Top 27 snapshot, and sends a Slack alert only when an SR enters or leaves the Top 27 before the next maintenance period. + +The alert includes the entered SRs, left SRs, current 27th/28th vote gap, and a Top 28 vote table. + +When `PHONE_ALERT_URL` is configured, the same Top 27 change also triggers a phone call escalation, independent of the Slack alert (a failure to reach it is logged but never blocks or fails the Slack notification). + ### Notifications This monitor only support java-tron node v4.8.1+, because of the API it used. diff --git a/tools/slack_sr_monitor/docker-compose.yml b/tools/slack_sr_monitor/docker-compose.yml index ff72702..cae336f 100644 --- a/tools/slack_sr_monitor/docker-compose.yml +++ b/tools/slack_sr_monitor/docker-compose.yml @@ -3,8 +3,25 @@ services: build: . container_name: slack-sr-monitor restart: unless-stopped + command: ["./slack_sr_monitor", "monitor"] environment: - SLACK_WEBHOOK=${SLACK_WEBHOOK} + - SLACK_SR_MONITOR_WEBHOOK=${SLACK_SR_MONITOR_WEBHOOK:-} + - TRON_NODES=${TRON_NODES:-} - TRON_NODE=${TRON_NODE:-https://api.trongrid.io} volumes: - ./logs:/home/monitor/logs + + slack-sr-guard: + build: . + container_name: slack-sr-guard + restart: unless-stopped + command: ["./slack_sr_monitor", "guard"] + environment: + - SLACK_WEBHOOK=${SLACK_WEBHOOK} + - SLACK_SR_GUARD_WEBHOOK=${SLACK_SR_GUARD_WEBHOOK:-} + - TRON_NODES=${TRON_NODES:-} + - TRON_NODE=${TRON_NODE:-https://api.trongrid.io} + - PHONE_ALERT_URL=${PHONE_ALERT_URL:-} + volumes: + - ./logs:/home/monitor/logs diff --git a/tools/slack_sr_monitor/main.go b/tools/slack_sr_monitor/main.go index 089b4b8..b8c9793 100644 --- a/tools/slack_sr_monitor/main.go +++ b/tools/slack_sr_monitor/main.go @@ -1,508 +1,87 @@ package main import ( - "bytes" - "encoding/hex" - "encoding/json" "fmt" "io" "log" - "net/http" "os" "path/filepath" "strings" "sync" - "time" "github.com/joho/godotenv" ) -const ( - DefaultTronNode = "https://api.trongrid.io" -) - -// Witness represents the structure of a witness returned by the Tron API -type Witness struct { - Address string `json:"address"` - VoteCount int64 `json:"voteCount"` - PubKey string `json:"pubKey"` - URL string `json:"url"` - TotalProduced int64 `json:"totalProduced"` - TotalMissed int64 `json:"totalMissed"` - LatestBlock int64 `json:"latestBlockNum"` - IsJobs bool `json:"isJobs"` - DisplayName string `json:"-"` -} - -func getAccountName(nodeURL string, address string) string { - url := fmt.Sprintf("%s/wallet/getaccount", nodeURL) - payload := map[string]interface{}{ - "address": address, - "visible": true, - } - jsonPayload, err := json.Marshal(payload) - if err != nil { - log.Printf("Error: failed to marshal account name payload for %s: %v", address, err) - return "" - } - - client := &http.Client{Timeout: 5 * time.Second} - resp, err := client.Post(url, "application/json", bytes.NewBuffer(jsonPayload)) - if err != nil { - log.Printf("Error: failed to request account name for %s: %v", address, err) - return "" - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - log.Printf("Error: failed to read account response body for %s: %v", address, err) - return "" - } - - var data map[string]interface{} - if err := json.Unmarshal(body, &data); err != nil { - log.Printf("Error: failed to unmarshal account JSON for %s: %v", address, err) - return "" - } - - if val, ok := data["account_name"]; ok { - if str, ok := val.(string); ok { - return str - } - return fmt.Sprintf("%v", val) - } - - return "" -} - -// WitnessListResponse is the wrapper for the witness list API response -type WitnessListResponse struct { - Witnesses []Witness `json:"witnesses"` -} - -// NextMaintenanceResponse is the wrapper for the maintenance time API response -type NextMaintenanceResponse struct { - Num int64 `json:"num"` -} - -func getNextMaintenanceTime(nodeURL string) (time.Time, error) { - url := fmt.Sprintf("%s/wallet/getnextmaintenancetime", nodeURL) - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Post(url, "application/json", nil) - if err != nil { - return time.Time{}, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return time.Time{}, fmt.Errorf("status code %d", resp.StatusCode) - } - - var result NextMaintenanceResponse - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return time.Time{}, err - } - - // Result is in milliseconds - return time.Unix(result.Num/1000, (result.Num%1000)*1000000), nil -} - -// NowBlockResponse captures the timestamp of the node's latest block. -type NowBlockResponse struct { - BlockHeader struct { - RawData struct { - Number int64 `json:"number"` - Timestamp int64 `json:"timestamp"` - } `json:"raw_data"` - } `json:"block_header"` -} - -// A fresh TRON node should always be within a few block intervals (block time is 3s). -// Anything beyond this threshold means the node is still catching up to the chain head. -const maxBlockAge = 30 * time.Second - -func getLatestBlockTime(nodeURL string) (time.Time, error) { - url := fmt.Sprintf("%s/wallet/getnowblock", nodeURL) - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Post(url, "application/json", nil) - if err != nil { - return time.Time{}, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return time.Time{}, fmt.Errorf("status code %d", resp.StatusCode) - } - - var result NowBlockResponse - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return time.Time{}, err - } - - ts := result.BlockHeader.RawData.Timestamp - return time.Unix(ts/1000, (ts%1000)*1000000), nil -} - -func getWitnessList(nodeURL string) ([]Witness, error) { - url := fmt.Sprintf("%s/wallet/getpaginatednowwitnesslist", nodeURL) - - payload := []byte(`{"offset": 0, "limit": 28, "visible": true}`) - - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Post(url, "application/json", bytes.NewBuffer(payload)) - if err != nil { - return nil, fmt.Errorf("failed to call Tron API: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("tron API returned status %d", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %v", err) - } - - var result WitnessListResponse - if err := json.Unmarshal(body, &result); err != nil { - return nil, fmt.Errorf("failed to unmarshal JSON: %v (body: %s)", err, string(body)) - } - - // For each witness, try to get the account name in parallel - log.Printf("Fetching account names for %d witnesses in parallel...\n", len(result.Witnesses)) - var wg sync.WaitGroup - for i := range result.Witnesses { - wg.Add(1) - go func(idx int) { - defer wg.Done() - w := &result.Witnesses[idx] - accName := getAccountName(nodeURL, w.Address) - if accName != "" { - if decoded, err := hex.DecodeString(accName); err == nil { - w.DisplayName = string(decoded) - } else { - w.DisplayName = accName - } - } - if w.DisplayName == "" { - w.DisplayName = w.URL - } - }(i) - } - wg.Wait() - - return result.Witnesses, nil -} - -func formatComma(n int64) string { - in := fmt.Sprintf("%d", n) - var out strings.Builder - if n < 0 { - out.WriteByte('-') - in = in[1:] - } - l := len(in) - for i, c := range in { - if i > 0 && (l-i)%3 == 0 { - out.WriteByte(',') - } - out.WriteRune(c) - } - return out.String() -} - -func sendToSlack(webhookURL string, witnesses []Witness, prevVotes map[string]int64, prevSRs []string) error { - var summary strings.Builder - summary.WriteString("*TRON SR Status Update (Maintenance Period)*") - summary.WriteString(fmt.Sprintf(" Time: %s\n", time.Now().UTC().Format(time.RFC1123))) - - // 1. Calculate SR changes for the summary - if len(prevSRs) > 0 { - currentTop27 := make(map[string]bool) - for i := 0; i < 27 && i < len(witnesses); i++ { - currentTop27[witnesses[i].Address] = true - } - - prevTop27 := make(map[string]bool) - for _, addr := range prevSRs { - prevTop27[addr] = true - } - - var entered []string - var left []string - - // Who enter - for i := 0; i < 27 && i < len(witnesses); i++ { - w := witnesses[i] - if !prevTop27[w.Address] { - name := w.DisplayName - if name == "" { - name = w.Address - } - entered = append(entered, name) - } - } - - // Who left - for _, addr := range prevSRs { - if !currentTop27[addr] { - name := addr - for _, w := range witnesses { - if w.Address == addr { - name = w.DisplayName - if name == "" { - name = w.Address - } - break - } - } - left = append(left, name) - } - } - - // Build rank map for previous SRs - prevRankMap := make(map[string]int) - for i, addr := range prevSRs { - prevRankMap[addr] = i + 1 // Rank is 1-based - } - - // Detect rank changes within top 27 - var rankChanges []string - for i := 0; i < 27 && i < len(witnesses); i++ { - w := witnesses[i] - if prevRank, ok := prevRankMap[w.Address]; ok { - change := prevRank - (i + 1) // Positive = rank up, negative = rank down - if change != 0 { - direction := "↑" - if change < 0 { - direction = "↓" - change = -change - } - name := w.DisplayName - if name == "" { - name = w.Address - } - rankChanges = append(rankChanges, - fmt.Sprintf("%s: %d → %d (%s%d)", name, prevRank, i+1, direction, change)) - } - } - } - - // Output all changes (entering, leaving, and rank changes) in one block - if len(entered) > 0 || len(left) > 0 || len(rankChanges) > 0 { - summary.WriteString("*SR Changes Detected:*\n") - if len(entered) > 0 { - summary.WriteString(fmt.Sprintf(">:inbox_tray: *Entered:* %s\n", strings.Join(entered, ", "))) - } - if len(left) > 0 { - summary.WriteString(fmt.Sprintf(">:outbox_tray: *Left:* %s\n", strings.Join(left, ", "))) - } - if len(rankChanges) > 0 { - summary.WriteString(">*Rank Changes:*\n") - for _, rc := range rankChanges { - summary.WriteString(fmt.Sprintf("> `%s`\n", rc)) - } - } - } else { - summary.WriteString("*Top 27 SRs remain unchanged.*\n") - } - } else { - summary.WriteString("*First check, initializing SR list*\n") - } - - // Calculate gap between 27th and 28th SR (always show this if we have enough witnesses) - if len(witnesses) >= 28 { - v27 := witnesses[26].VoteCount - v28 := witnesses[27].VoteCount - gap := v27 - v28 - summary.WriteString(fmt.Sprintf("*Gap (27 vs 28):* `%s` votes", formatComma(gap))) - } - - // 2. Build detailed list for attachment - var details strings.Builder - details.WriteString("```\n") - for i, w := range witnesses { - name := w.DisplayName - if name == "" { - name = w.Address - } - - prev := prevVotes[w.Address] - diff := w.VoteCount - prev - - diffStr := formatComma(diff) - if diff >= 0 { - diffStr = "+" + diffStr - } - if prev == 0 { - diffStr = "-" - } - - details.WriteString(fmt.Sprintf("%2d. %-25s Current: %15s Change: %10s\n", - i+1, name, formatComma(w.VoteCount), diffStr)) - } - details.WriteString("```") - - // 3. Construct Payload with Attachments - payload := map[string]interface{}{ - "text": summary.String(), - "attachments": []map[string]interface{}{ - { - "text": details.String(), - "color": "#36a64f", - }, - }, - } - - jsonPayload, err := json.Marshal(payload) - if err != nil { - return fmt.Errorf("failed to marshal slack payload: %v", err) - } - - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) - if err != nil { - return fmt.Errorf("failed to send to slack: %v", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) - } - - return nil -} - -func updateLastStatus(witnesses []Witness, lastVotes map[string]int64) []string { - var top27 []string - for i, w := range witnesses { - lastVotes[w.Address] = w.VoteCount - if i < 27 { - top27 = append(top27, w.Address) - } - } - return top27 -} - -// The final path segment of a Slack webhook URL is the secret token; mask it before logging. -func maskWebhook(url string) string { - idx := strings.LastIndex(url, "/") - if idx < 0 || idx == len(url)-1 { - return "***" - } - return url[:idx+1] + "***" -} - -func main() { - // Ensure logs directory exists +func setupLogging() { logDir := "logs" if _, err := os.Stat(logDir); os.IsNotExist(err) { _ = os.Mkdir(logDir, 0755) } - // Setup logging to both file and stdout - logPath := filepath.Join(logDir, "sr_monitor.log") + logPath := filepath.Join(logDir, "slack_sr_tools.log") logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) if err != nil { fmt.Printf("Error opening log file: %v, falling back to stdout only\n", err) } else { - mw := io.MultiWriter(os.Stdout, logFile) - log.SetOutput(mw) + log.SetOutput(io.MultiWriter(os.Stdout, logFile)) } log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) +} + +func main() { + setupLogging() - // Load .env file if err := godotenv.Load(); err != nil { log.Println("Warning: No .env file found, using system environment variables") } - tronNode := os.Getenv("TRON_NODE") - if tronNode == "" { - tronNode = DefaultTronNode - } - - slackWebhook := os.Getenv("SLACK_WEBHOOK") - if slackWebhook == "" { - log.Println("Error: SLACK_WEBHOOK environment variable is not set") - log.Println("Usage: SLACK_WEBHOOK=https://hooks.slack.com/... [TRON_NODE=...] go run main.go") + tronNodes := parseTronNodes() + mode := "both" + if len(os.Args) > 1 { + mode = strings.ToLower(os.Args[1]) + } + + switch mode { + case "monitor", "sr-monitor", "slack-sr-monitor": + slackWebhook := getSlackWebhook("SLACK_SR_MONITOR_WEBHOOK") + if slackWebhook == "" { + log.Println("Error: SLACK_SR_MONITOR_WEBHOOK or SLACK_WEBHOOK environment variable is not set") + log.Println("Usage: SLACK_SR_MONITOR_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run . monitor") + os.Exit(1) + } + runSRMonitor(tronNodes, slackWebhook) + case "guard", "sr-guard", "slack-sr-guard": + slackWebhook := getSlackWebhook("SLACK_SR_GUARD_WEBHOOK") + if slackWebhook == "" { + log.Println("Error: SLACK_SR_GUARD_WEBHOOK or SLACK_WEBHOOK environment variable is not set") + log.Println("Usage: SLACK_SR_GUARD_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run . guard") + os.Exit(1) + } + runSRGuard(tronNodes, slackWebhook, getPhoneAlertURL()) + case "both", "": + monitorWebhook := getSlackWebhook("SLACK_SR_MONITOR_WEBHOOK") + guardWebhook := getSlackWebhook("SLACK_SR_GUARD_WEBHOOK") + if monitorWebhook == "" || guardWebhook == "" { + log.Println("Error: SLACK_SR_MONITOR_WEBHOOK/SLACK_SR_GUARD_WEBHOOK or SLACK_WEBHOOK environment variable is not set") + log.Println("Usage: SLACK_WEBHOOK=https://hooks.slack.com/... [TRON_NODES=...] go run .") + os.Exit(1) + } + phoneAlertURL := getPhoneAlertURL() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + runSRMonitor(tronNodes, monitorWebhook) + }() + go func() { + defer wg.Done() + runSRGuard(tronNodes, guardWebhook, phoneAlertURL) + }() + wg.Wait() + default: + log.Printf("Error: unknown mode %q", mode) + log.Println("Usage: go run . [monitor|guard|both]") os.Exit(1) } - - log.Printf("Starting SR monitor.\nNode: %s\nSlack Webhook: %s\n", tronNode, maskWebhook(slackWebhook)) - - // Map to track votes: Address -> VoteCount - lastVotes := make(map[string]int64) - var lastTop27 []string - - // Initial check - log.Println("Performing initial check...") - witnesses, err := getWitnessList(tronNode) - if err != nil { - log.Printf("Initial check failed: %v\n", err) - } else { - log.Printf("Successfully fetched %d witnesses. Sending to Slack...\n", len(witnesses)) - if err := sendToSlack(slackWebhook, witnesses, lastVotes, lastTop27); err != nil { - log.Printf("Failed to send initial update to Slack: %v\n", err) - } else { - lastTop27 = updateLastStatus(witnesses, lastVotes) - } - } - - log.Println("Monitoring for maintenance periods via getnextmaintenancetime...") - - for { - nextTime, err := getNextMaintenanceTime(tronNode) - if err != nil { - log.Printf("Error fetching next maintenance time: %v, retrying in 1 minute...\n", err) - time.Sleep(1 * time.Minute) - continue - } - - // A node that is still catching up will report a past maintenance time and would - // otherwise cause this loop to send duplicate Slack messages every 2 minutes. - blockTime, err := getLatestBlockTime(tronNode) - if err != nil { - log.Printf("Error fetching latest block: %v, retrying in 1 minute...\n", err) - time.Sleep(1 * time.Minute) - continue - } - if age := time.Since(blockTime); age > maxBlockAge { - log.Printf("Node is %s behind chain head (latest block at %s), waiting 2 minutes...\n", - age.Truncate(time.Second), blockTime.UTC().Format(time.RFC3339)) - time.Sleep(2 * time.Minute) - continue - } - - // Calculate trigger time: next maintenance time + 1 minute - triggerTime := nextTime.Add(1 * time.Minute) - now := time.Now().UTC() - waitDuration := triggerTime.Sub(now) - - if waitDuration > 0 { - log.Printf("Next maintenance time: %s (UTC). Waiting %v until %s...\n", - nextTime.Format(time.RFC1123), waitDuration.Truncate(time.Second), triggerTime.Format(time.RFC1123)) - time.Sleep(waitDuration) - } else { - log.Printf("Maintenance time %s has already passed. Checking now...\n", nextTime.Format(time.RFC1123)) - } - - log.Printf("[%s] Maintenance period reached (+1m). Fetching SR list...\n", time.Now().UTC().Format(time.RFC3339)) - witnesses, err := getWitnessList(tronNode) - if err != nil { - log.Printf("Error fetching witness list: %v\n", err) - } else { - if err := sendToSlack(slackWebhook, witnesses, lastVotes, lastTop27); err != nil { - log.Printf("Error sending to Slack: %v\n", err) - } else { - log.Println("Successfully sent SR list to Slack") - lastTop27 = updateLastStatus(witnesses, lastVotes) - } - } - - // Wait a bit before checking for the NEXT maintenance time to avoid double-triggering - time.Sleep(2 * time.Minute) - } } diff --git a/tools/slack_sr_monitor/sr_guard.go b/tools/slack_sr_monitor/sr_guard.go new file mode 100644 index 0000000..52bea41 --- /dev/null +++ b/tools/slack_sr_monitor/sr_guard.go @@ -0,0 +1,412 @@ +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const ( + pollInterval = time.Minute + topSRLimit = 27 + fetchLimit = 28 + guardStatePath = "logs/sr_guard_state.json" + guardStateVersion = 1 +) + +type guardState struct { + Version int `json:"version"` + UpdatedAt time.Time `json:"updated_at"` + PreviousTop27 []string `json:"previous_top_27"` + NameCache map[string]string `json:"name_cache"` +} + +func getGuardWitnessListFromNodes(nodeURLs []string) ([]Witness, error) { + payload := []byte(fmt.Sprintf(`{"offset":0,"limit":%d,"visible":true}`, fetchLimit)) + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getpaginatednowwitnesslist", payload, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var result WitnessListResponse + if err := json.Unmarshal(body, &result); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal JSON: %v", nodeURL, err)) + continue + } + + return result.Witnesses, nil + } + + return nil, fmt.Errorf("all TRON nodes failed for /wallet/getpaginatednowwitnesslist: %s", strings.Join(errors, "; ")) +} + +func getGuardAccountNameFromNodes(nodeURLs []string, address string) string { + payload := map[string]interface{}{ + "address": address, + "visible": true, + } + jsonPayload, err := json.Marshal(payload) + if err != nil { + log.Printf("failed to marshal account name payload for %s: %v", address, err) + return "" + } + + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getaccount", jsonPayload, 5*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + name, ok := parseAccountName(address, body) + if !ok { + errors = append(errors, fmt.Sprintf("%s: invalid account JSON", nodeURL)) + continue + } + + return name + } + + log.Printf("failed to request account name for %s: %s", address, strings.Join(errors, "; ")) + return "" +} + +func parseAccountName(address string, body []byte) (string, bool) { + var data map[string]interface{} + if err := json.Unmarshal(body, &data); err != nil { + log.Printf("failed to unmarshal account JSON for %s: %v", address, err) + return "", false + } + + val, ok := data["account_name"] + if !ok { + return "", true + } + + str, ok := val.(string) + if !ok { + str = fmt.Sprintf("%v", val) + } + decoded, err := hex.DecodeString(str) + if err != nil { + return str, true + } + return string(decoded), true +} + +func fillDisplayNames(nodeURLs []string, witnesses []Witness, nameCache map[string]string) { + var wg sync.WaitGroup + var mu sync.Mutex + + for i := range witnesses { + if name, ok := nameCache[witnesses[i].Address]; ok { + witnesses[i].DisplayName = name + continue + } + + wg.Add(1) + go func(idx int) { + defer wg.Done() + + witness := witnesses[idx] + name := getGuardAccountNameFromNodes(nodeURLs, witness.Address) + if name == "" { + name = witness.URL + } + if name == "" { + name = witness.Address + } + + mu.Lock() + nameCache[witness.Address] = name + witnesses[idx].DisplayName = name + mu.Unlock() + }(i) + } + + wg.Wait() +} + +func topAddresses(witnesses []Witness) []string { + limit := topSRLimit + if len(witnesses) < limit { + limit = len(witnesses) + } + + addresses := make([]string, 0, limit) + for i := 0; i < limit; i++ { + addresses = append(addresses, witnesses[i].Address) + } + return addresses +} + +func findTopSRChanges(previous []string, current []Witness) ([]Witness, []string) { + previousSet := make(map[string]bool, len(previous)) + for _, address := range previous { + previousSet[address] = true + } + + currentSet := make(map[string]bool, topSRLimit) + var entered []Witness + for i := 0; i < topSRLimit && i < len(current); i++ { + witness := current[i] + currentSet[witness.Address] = true + if !previousSet[witness.Address] { + entered = append(entered, witness) + } + } + + var left []string + for _, address := range previous { + if !currentSet[address] { + left = append(left, address) + } + } + + return entered, left +} + +func witnessName(w Witness) string { + if w.DisplayName != "" { + return w.DisplayName + } + if w.URL != "" { + return w.URL + } + return w.Address +} + +func sendAlert(webhookURL string, witnesses []Witness, entered []Witness, left []string, nameCache map[string]string) error { + var text strings.Builder + text.WriteString("*:warning: TRON Top 27 SR Change Detected*\n") + text.WriteString(fmt.Sprintf("Time: %s\n", time.Now().UTC().Format(time.RFC1123))) + + if len(entered) > 0 { + var names []string + for _, witness := range entered { + names = append(names, fmt.Sprintf("%s (`%s` votes)", witnessName(witness), formatComma(witness.VoteCount))) + } + text.WriteString(fmt.Sprintf(">:inbox_tray: *Entered Top 27:* %s\n", strings.Join(names, ", "))) + } + + if len(left) > 0 { + var names []string + for _, address := range left { + name := nameCache[address] + if name == "" { + name = address + } + names = append(names, name) + } + text.WriteString(fmt.Sprintf(">:outbox_tray: *Left Top 27:* %s\n", strings.Join(names, ", "))) + } + + var details strings.Builder + details.WriteString("```\n") + for i, witness := range witnesses { + details.WriteString(fmt.Sprintf("%2d. %-25s %15s\n", i+1, witnessName(witness), formatComma(witness.VoteCount))) + } + details.WriteString("```") + + payload := map[string]interface{}{ + "text": text.String(), + "attachments": []map[string]interface{}{ + { + "text": details.String(), + "color": "#e01e5a", + }, + }, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send to slack: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// buildPhoneAlertSummary produces a plain-text (no Slack markdown) summary, describing +// which SRs entered/left the Top 27, for the phone alert message. +func buildPhoneAlertSummary(entered []Witness, left []string, nameCache map[string]string) string { + var parts []string + + if len(entered) > 0 { + var names []string + for _, witness := range entered { + names = append(names, witnessName(witness)) + } + parts = append(parts, fmt.Sprintf("entered: %s", strings.Join(names, ", "))) + } + + if len(left) > 0 { + var names []string + for _, address := range left { + name := nameCache[address] + if name == "" { + name = address + } + names = append(names, name) + } + parts = append(parts, fmt.Sprintf("left: %s", strings.Join(names, ", "))) + } + + return "TRON Top 27 SR change detected (" + strings.Join(parts, "; ") + ")" +} + +// triggerPhoneAlert posts a short message to a webhook that triggers a phone/voice +// call alert after the Slack alert has been delivered successfully. +func triggerPhoneAlert(phoneAlertURL string, entered []Witness, left []string, nameCache map[string]string) error { + summary := buildPhoneAlertSummary(entered, left, nameCache) + + payload := map[string]string{ + "message": summary, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal phone alert payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(phoneAlertURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send phone alert: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("phone alert endpoint returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func loadGuardState() guardState { + state := guardState{ + Version: guardStateVersion, + NameCache: make(map[string]string), + } + + body, err := os.ReadFile(guardStatePath) + if err != nil { + if !os.IsNotExist(err) { + log.Printf("failed to read guard state: %v", err) + } + return state + } + + if err := json.Unmarshal(body, &state); err != nil { + log.Printf("failed to parse guard state: %v", err) + return guardState{Version: guardStateVersion, NameCache: make(map[string]string)} + } + if state.NameCache == nil { + state.NameCache = make(map[string]string) + } + log.Printf("loaded guard state: top27=%d names=%d updated_at=%s", len(state.PreviousTop27), len(state.NameCache), state.UpdatedAt.UTC().Format(time.RFC3339)) + return state +} + +func saveGuardState(previousTop27 []string, nameCache map[string]string) { + state := guardState{ + Version: guardStateVersion, + UpdatedAt: time.Now().UTC(), + PreviousTop27: previousTop27, + NameCache: nameCache, + } + body, err := json.MarshalIndent(state, "", " ") + if err != nil { + log.Printf("failed to marshal guard state: %v", err) + return + } + if err := os.MkdirAll(filepath.Dir(guardStatePath), 0755); err != nil { + log.Printf("failed to create guard state directory: %v", err) + return + } + if err := os.WriteFile(guardStatePath, body, 0644); err != nil { + log.Printf("failed to write guard state: %v", err) + } +} + +func runSRGuard(tronNodes []string, slackWebhook string, phoneAlertURL string) { + phoneLog := "disabled" + if phoneAlertURL != "" { + phoneLog = maskWebhook(phoneAlertURL) + } + log.Printf("Starting SR guard. Nodes: %s Slack Webhook: %s Phone Alert URL: %s Poll interval: %s", + strings.Join(tronNodes, ", "), maskWebhook(slackWebhook), phoneLog, pollInterval) + + state := loadGuardState() + nameCache := state.NameCache + previousTop27 := state.PreviousTop27 + ticker := time.NewTicker(pollInterval) + defer ticker.Stop() + + for { + witnesses, err := getGuardWitnessListFromNodes(tronNodes) + if err != nil { + log.Printf("failed to fetch witness list: %v", err) + } else { + fillDisplayNames(tronNodes, witnesses, nameCache) + + if len(previousTop27) == 0 { + previousTop27 = topAddresses(witnesses) + saveGuardState(previousTop27, nameCache) + log.Printf("initialized Top 27 snapshot with %d witnesses", len(previousTop27)) + } else { + entered, left := findTopSRChanges(previousTop27, witnesses) + if len(entered) > 0 || len(left) > 0 { + log.Printf("Top 27 change detected: entered=%d left=%d", len(entered), len(left)) + + if err := sendAlert(slackWebhook, witnesses, entered, left, nameCache); err != nil { + log.Printf("failed to send Slack alert: %v", err) + } else { + previousTop27 = topAddresses(witnesses) + saveGuardState(previousTop27, nameCache) + log.Println("sent Slack alert and updated Top 27 snapshot") + + if phoneAlertURL != "" { + if err := triggerPhoneAlert(phoneAlertURL, entered, left, nameCache); err != nil { + log.Printf("failed to trigger phone alert: %v", err) + } else { + log.Println("triggered phone alert") + } + } + } + } else { + previousTop27 = topAddresses(witnesses) + saveGuardState(previousTop27, nameCache) + log.Println("Top 27 unchanged") + } + } + } + + <-ticker.C + } +} diff --git a/tools/slack_sr_monitor/sr_guard_phone_test.go b/tools/slack_sr_monitor/sr_guard_phone_test.go new file mode 100644 index 0000000..4982cab --- /dev/null +++ b/tools/slack_sr_monitor/sr_guard_phone_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestTriggerPhoneAlert(t *testing.T) { + var gotBody map[string]interface{} + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("failed to decode request body: %v", err) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + entered := []Witness{{Address: "TEntered1", DisplayName: "New SR", VoteCount: 1000}} + left := []string{"TLeft1"} + nameCache := map[string]string{"TLeft1": "Old SR"} + + if err := triggerPhoneAlert(server.URL, entered, left, nameCache); err != nil { + t.Fatalf("triggerPhoneAlert returned error: %v", err) + } + + message, _ := gotBody["message"].(string) + if message == "" { + t.Fatalf("expected non-empty message field") + } + if !strings.Contains(message, "New SR") { + t.Errorf("expected message to mention entered witness 'New SR', got %q", message) + } + if !strings.Contains(message, "Old SR") { + t.Errorf("expected message to mention left witness 'Old SR', got %q", message) + } +} diff --git a/tools/slack_sr_monitor/sr_monitor.go b/tools/slack_sr_monitor/sr_monitor.go new file mode 100644 index 0000000..39c4ab7 --- /dev/null +++ b/tools/slack_sr_monitor/sr_monitor.go @@ -0,0 +1,667 @@ +package main + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const ( + DefaultTronNode = "https://api.trongrid.io" + monitorStatePath = "logs/sr_monitor_state.json" + monitorStateVersion = 1 +) + +type monitorState struct { + Version int `json:"version"` + UpdatedAt time.Time `json:"updated_at"` + LastVotes map[string]int64 `json:"last_votes"` + LastTop27 []string `json:"last_top_27"` +} + +// Witness represents the structure of a witness returned by the Tron API +type Witness struct { + Address string `json:"address"` + VoteCount int64 `json:"voteCount"` + PubKey string `json:"pubKey"` + URL string `json:"url"` + TotalProduced int64 `json:"totalProduced"` + TotalMissed int64 `json:"totalMissed"` + LatestBlock int64 `json:"latestBlockNum"` + IsJobs bool `json:"isJobs"` + DisplayName string `json:"-"` +} + +func getAccountNameFromNodes(nodeURLs []string, address string) string { + payload := map[string]interface{}{ + "address": address, + "visible": true, + } + jsonPayload, err := json.Marshal(payload) + if err != nil { + log.Printf("Error: failed to marshal account name payload for %s: %v", address, err) + return "" + } + + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getaccount", jsonPayload, 5*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var data map[string]interface{} + if err := json.Unmarshal(body, &data); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal account JSON: %v", nodeURL, err)) + continue + } + + if val, ok := data["account_name"]; ok { + if str, ok := val.(string); ok { + return str + } + return fmt.Sprintf("%v", val) + } + + return "" + } + + log.Printf("Error: failed to request account name for %s: %s", address, strings.Join(errors, "; ")) + return "" +} + +func parseTronNodes() []string { + raw := os.Getenv("TRON_NODES") + if raw == "" { + raw = os.Getenv("TRON_NODE") + } + if raw == "" { + raw = DefaultTronNode + } + + var nodes []string + for _, node := range strings.Split(raw, ",") { + node = strings.TrimSpace(node) + if node != "" { + nodes = append(nodes, strings.TrimRight(node, "/")) + } + } + if len(nodes) == 0 { + return []string{DefaultTronNode} + } + return nodes +} + +func getSlackWebhook(specificEnv string) string { + if webhook := os.Getenv(specificEnv); webhook != "" { + return webhook + } + return os.Getenv("SLACK_WEBHOOK") +} + +// getPhoneAlertURL returns the Phone_AWS API Gateway endpoint used by slack_sr_guard to +// trigger a phone call directly. Optional; when unset, the phone alert is skipped. +func getPhoneAlertURL() string { + return os.Getenv("PHONE_ALERT_URL") +} + +func postToTron(nodeURLs []string, path string, payload []byte, timeout time.Duration) ([]byte, string, error) { + client := &http.Client{Timeout: timeout} + var errors []string + + for _, nodeURL := range nodeURLs { + resp, err := client.Post(nodeURL+path, "application/json", bytes.NewBuffer(payload)) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", nodeURL, err)) + continue + } + + body, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + if readErr != nil { + errors = append(errors, fmt.Sprintf("%s: failed to read response body: %v", nodeURL, readErr)) + continue + } + if resp.StatusCode != http.StatusOK { + errors = append(errors, fmt.Sprintf("%s: status %d: %s", nodeURL, resp.StatusCode, string(body))) + continue + } + + return body, nodeURL, nil + } + + return nil, "", fmt.Errorf("all TRON nodes failed for %s: %s", path, strings.Join(errors, "; ")) +} + +// WitnessListResponse is the wrapper for the witness list API response +type WitnessListResponse struct { + Witnesses []Witness `json:"witnesses"` +} + +// NextMaintenanceResponse is the wrapper for the maintenance time API response +type NextMaintenanceResponse struct { + Num int64 `json:"num"` +} + +func getNextMaintenanceTimeFromNodes(nodeURLs []string) (time.Time, error) { + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getnextmaintenancetime", nil, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var result NextMaintenanceResponse + if err := json.Unmarshal(body, &result); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal next maintenance JSON: %v", nodeURL, err)) + continue + } + + return time.Unix(result.Num/1000, (result.Num%1000)*1000000), nil + } + + return time.Time{}, fmt.Errorf("all TRON nodes failed for /wallet/getnextmaintenancetime: %s", strings.Join(errors, "; ")) +} + +// NowBlockResponse captures the timestamp of the node's latest block. +type NowBlockResponse struct { + BlockHeader struct { + RawData struct { + Number int64 `json:"number"` + Timestamp int64 `json:"timestamp"` + } `json:"raw_data"` + } `json:"block_header"` +} + +// A fresh TRON node should always be within a few block intervals (block time is 3s). +// Anything beyond this threshold means the node is still catching up to the chain head. +const ( + maxBlockAge = 30 * time.Second + freshnessRetryInterval = 2 * time.Minute + freshnessRetryTimeout = 2 * time.Hour + witnessRetryInterval = 30 * time.Second + witnessRetryTimeout = 10 * time.Minute +) + +func getLatestBlockTimeFromNodes(nodeURLs []string) (time.Time, error) { + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getnowblock", nil, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + var result NowBlockResponse + if err := json.Unmarshal(body, &result); err != nil { + errors = append(errors, fmt.Sprintf("%s: failed to unmarshal now block JSON: %v", nodeURL, err)) + continue + } + + ts := result.BlockHeader.RawData.Timestamp + return time.Unix(ts/1000, (ts%1000)*1000000), nil + } + + return time.Time{}, fmt.Errorf("all TRON nodes failed for /wallet/getnowblock: %s", strings.Join(errors, "; ")) +} + +func getWitnessListFromNodes(nodeURLs []string) ([]Witness, error) { + payload := []byte(`{"offset": 0, "limit": 28, "visible": true}`) + var errors []string + for _, nodeURL := range nodeURLs { + body, _, err := postToTron([]string{nodeURL}, "/wallet/getpaginatednowwitnesslist", payload, 10*time.Second) + if err != nil { + errors = append(errors, err.Error()) + continue + } + + witnesses, err := parseWitnessList(nodeURLs, body) + if err != nil { + errors = append(errors, fmt.Sprintf("%s: %v", nodeURL, err)) + continue + } + + return witnesses, nil + } + + return nil, fmt.Errorf("all TRON nodes failed for /wallet/getpaginatednowwitnesslist: %s", strings.Join(errors, "; ")) +} + +func waitForFreshNode(nodeURLs []string) error { + deadline := time.Now().Add(freshnessRetryTimeout) + + for { + blockTime, err := getLatestBlockTimeFromNodes(nodeURLs) + if err != nil { + log.Printf("Error fetching latest block: %v", err) + } else if age := time.Since(blockTime); age <= maxBlockAge { + log.Printf("Node is fresh (latest block at %s, age %s)", blockTime.UTC().Format(time.RFC3339), age.Truncate(time.Second)) + return nil + } else { + log.Printf("Node is %s behind chain head (latest block at %s), waiting %s...", + age.Truncate(time.Second), blockTime.UTC().Format(time.RFC3339), freshnessRetryInterval) + } + + if time.Now().Add(freshnessRetryInterval).After(deadline) { + return fmt.Errorf("node did not become fresh within %s", freshnessRetryTimeout) + } + time.Sleep(freshnessRetryInterval) + } +} + +func getWitnessListWithRetry(nodeURLs []string) ([]Witness, error) { + deadline := time.Now().Add(witnessRetryTimeout) + + for { + witnesses, err := getWitnessListFromNodes(nodeURLs) + if err == nil { + return witnesses, nil + } + + log.Printf("Error fetching witness list: %v", err) + if time.Now().Add(witnessRetryInterval).After(deadline) { + return nil, fmt.Errorf("failed to fetch witness list within %s: %v", witnessRetryTimeout, err) + } + log.Printf("Retrying witness list in %s...", witnessRetryInterval) + time.Sleep(witnessRetryInterval) + } +} + +func parseWitnessList(nodeURLs []string, body []byte) ([]Witness, error) { + var result WitnessListResponse + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON: %v (body: %s)", err, string(body)) + } + + // For each witness, try to get the account name in parallel + log.Printf("Fetching account names for %d witnesses in parallel...\n", len(result.Witnesses)) + var wg sync.WaitGroup + for i := range result.Witnesses { + wg.Add(1) + go func(idx int) { + defer wg.Done() + w := &result.Witnesses[idx] + accName := getAccountNameFromNodes(nodeURLs, w.Address) + if accName != "" { + if decoded, err := hex.DecodeString(accName); err == nil { + w.DisplayName = string(decoded) + } else { + w.DisplayName = accName + } + } + if w.DisplayName == "" { + w.DisplayName = w.URL + } + }(i) + } + wg.Wait() + + return result.Witnesses, nil +} + +func formatComma(n int64) string { + in := fmt.Sprintf("%d", n) + var out strings.Builder + if n < 0 { + out.WriteByte('-') + in = in[1:] + } + l := len(in) + for i, c := range in { + if i > 0 && (l-i)%3 == 0 { + out.WriteByte(',') + } + out.WriteRune(c) + } + return out.String() +} + +func sendToSlack(webhookURL string, nodeURLs []string, witnesses []Witness, prevVotes map[string]int64, prevSRs []string) error { + var summary strings.Builder + summary.WriteString("*TRON SR Status Update (Maintenance Period)*") + summary.WriteString(fmt.Sprintf(" Time: %s\n", time.Now().UTC().Format(time.RFC1123))) + + // 1. Calculate SR changes for the summary + if len(prevSRs) > 0 { + currentTop27 := make(map[string]bool) + for i := 0; i < 27 && i < len(witnesses); i++ { + currentTop27[witnesses[i].Address] = true + } + + prevTop27 := make(map[string]bool) + for _, addr := range prevSRs { + prevTop27[addr] = true + } + + var entered []string + var left []string + + // Who enter + for i := 0; i < 27 && i < len(witnesses); i++ { + w := witnesses[i] + if !prevTop27[w.Address] { + name := w.DisplayName + if name == "" { + name = w.Address + } + entered = append(entered, name) + } + } + + // Who left + for _, addr := range prevSRs { + if !currentTop27[addr] { + name := addr + foundInCurrentList := false + for _, w := range witnesses { + if w.Address == addr { + foundInCurrentList = true + name = w.DisplayName + if name == "" { + name = w.Address + } + break + } + } + if !foundInCurrentList { + if accountName := getAccountNameFromNodes(nodeURLs, addr); accountName != "" { + name = accountName + } + } + left = append(left, name) + } + } + + // Build rank map for previous SRs + prevRankMap := make(map[string]int) + for i, addr := range prevSRs { + prevRankMap[addr] = i + 1 // Rank is 1-based + } + + // Detect rank changes within top 27 + var rankChanges []string + for i := 0; i < 27 && i < len(witnesses); i++ { + w := witnesses[i] + if prevRank, ok := prevRankMap[w.Address]; ok { + change := prevRank - (i + 1) // Positive = rank up, negative = rank down + if change != 0 { + direction := "↑" + if change < 0 { + direction = "↓" + change = -change + } + name := w.DisplayName + if name == "" { + name = w.Address + } + rankChanges = append(rankChanges, + fmt.Sprintf("%s: %d → %d (%s%d)", name, prevRank, i+1, direction, change)) + } + } + } + + // Output all changes (entering, leaving, and rank changes) in one block + if len(entered) > 0 || len(left) > 0 || len(rankChanges) > 0 { + summary.WriteString("*SR Changes Detected:*\n") + if len(entered) > 0 { + summary.WriteString(fmt.Sprintf(">:inbox_tray: *Entered:* %s\n", strings.Join(entered, ", "))) + } + if len(left) > 0 { + summary.WriteString(fmt.Sprintf(">:outbox_tray: *Left:* %s\n", strings.Join(left, ", "))) + } + if len(rankChanges) > 0 { + summary.WriteString(">*Rank Changes:*\n") + for _, rc := range rankChanges { + summary.WriteString(fmt.Sprintf("> `%s`\n", rc)) + } + } + } else { + summary.WriteString("*Top 27 SRs remain unchanged.*\n") + } + } else { + summary.WriteString("*First check, initializing SR list*\n") + } + + // Calculate gap between 27th and 28th SR (always show this if we have enough witnesses) + if len(witnesses) >= 28 { + v27 := witnesses[26].VoteCount + v28 := witnesses[27].VoteCount + gap := v27 - v28 + summary.WriteString(fmt.Sprintf("*Gap (27 vs 28):* `%s` votes", formatComma(gap))) + } + + // 2. Build detailed list for attachment + var details strings.Builder + details.WriteString("```\n") + for i, w := range witnesses { + name := w.DisplayName + if name == "" { + name = w.Address + } + + prev := prevVotes[w.Address] + diff := w.VoteCount - prev + + diffStr := formatComma(diff) + if diff >= 0 { + diffStr = "+" + diffStr + } + if prev == 0 { + diffStr = "-" + } + + details.WriteString(fmt.Sprintf("%2d. %-25s Current: %15s Change: %10s\n", + i+1, name, formatComma(w.VoteCount), diffStr)) + } + details.WriteString("```") + + // 3. Construct Payload with Attachments + payload := map[string]interface{}{ + "text": summary.String(), + "attachments": []map[string]interface{}{ + { + "text": details.String(), + "color": "#36a64f", + }, + }, + } + + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send to slack: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func sendSlackText(webhookURL string, text string) error { + payload := map[string]interface{}{ + "text": text, + } + jsonPayload, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal slack payload: %v", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Post(webhookURL, "application/json", bytes.NewBuffer(jsonPayload)) + if err != nil { + return fmt.Errorf("failed to send to slack: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("slack returned status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +func updateLastStatus(witnesses []Witness, lastVotes map[string]int64) []string { + var top27 []string + for i, w := range witnesses { + lastVotes[w.Address] = w.VoteCount + if i < 27 { + top27 = append(top27, w.Address) + } + } + return top27 +} + +func loadMonitorState() monitorState { + state := monitorState{ + Version: monitorStateVersion, + LastVotes: make(map[string]int64), + } + + body, err := os.ReadFile(monitorStatePath) + if err != nil { + if !os.IsNotExist(err) { + log.Printf("Failed to read monitor state: %v", err) + } + return state + } + + if err := json.Unmarshal(body, &state); err != nil { + log.Printf("Failed to parse monitor state: %v", err) + return monitorState{Version: monitorStateVersion, LastVotes: make(map[string]int64)} + } + if state.LastVotes == nil { + state.LastVotes = make(map[string]int64) + } + log.Printf("Loaded monitor state: votes=%d top27=%d updated_at=%s", len(state.LastVotes), len(state.LastTop27), state.UpdatedAt.UTC().Format(time.RFC3339)) + return state +} + +func saveMonitorState(lastVotes map[string]int64, lastTop27 []string) { + state := monitorState{ + Version: monitorStateVersion, + UpdatedAt: time.Now().UTC(), + LastVotes: lastVotes, + LastTop27: lastTop27, + } + body, err := json.MarshalIndent(state, "", " ") + if err != nil { + log.Printf("Failed to marshal monitor state: %v", err) + return + } + if err := os.MkdirAll(filepath.Dir(monitorStatePath), 0755); err != nil { + log.Printf("Failed to create monitor state directory: %v", err) + return + } + if err := os.WriteFile(monitorStatePath, body, 0644); err != nil { + log.Printf("Failed to write monitor state: %v", err) + } +} + +// The final path segment of a Slack webhook URL is the secret token; mask it before logging. +func maskWebhook(url string) string { + idx := strings.LastIndex(url, "/") + if idx < 0 || idx == len(url)-1 { + return "***" + } + return url[:idx+1] + "***" +} + +func runSRMonitor(tronNodes []string, slackWebhook string) { + log.Printf("Starting SR monitor.\nNodes: %s\nSlack Webhook: %s\n", strings.Join(tronNodes, ", "), maskWebhook(slackWebhook)) + + state := loadMonitorState() + lastVotes := state.LastVotes + lastTop27 := state.LastTop27 + + if len(lastVotes) > 0 && len(lastTop27) > 0 { + text := fmt.Sprintf("TRON SR monitor started with existing state. Previous state updated at `%s`. Next report will be sent after the next maintenance period.", state.UpdatedAt.UTC().Format(time.RFC3339)) + if err := sendSlackText(slackWebhook, text); err != nil { + log.Printf("Failed to send startup log to Slack: %v\n", err) + } else { + log.Println("Sent startup log to Slack; keeping existing monitor state until next maintenance report") + } + } else { + log.Println("No complete monitor state found. Performing initial check...") + witnesses, err := getWitnessListFromNodes(tronNodes) + if err != nil { + log.Printf("Initial check failed: %v\n", err) + } else { + log.Printf("Successfully fetched %d witnesses. Sending to Slack...\n", len(witnesses)) + if err := sendToSlack(slackWebhook, tronNodes, witnesses, lastVotes, lastTop27); err != nil { + log.Printf("Failed to send initial update to Slack: %v\n", err) + } else { + lastTop27 = updateLastStatus(witnesses, lastVotes) + saveMonitorState(lastVotes, lastTop27) + } + } + } + + log.Println("Monitoring for maintenance periods via getnextmaintenancetime...") + + for { + nextTime, err := getNextMaintenanceTimeFromNodes(tronNodes) + if err != nil { + log.Printf("Error fetching next maintenance time: %v, retrying in 1 minute...\n", err) + time.Sleep(1 * time.Minute) + continue + } + + // Calculate trigger time: next maintenance time + 1 minute + triggerTime := nextTime.Add(1 * time.Minute) + now := time.Now().UTC() + waitDuration := triggerTime.Sub(now) + + if waitDuration > 0 { + log.Printf("Next maintenance time: %s (UTC). Waiting %v until %s...\n", + nextTime.Format(time.RFC1123), waitDuration.Truncate(time.Second), triggerTime.Format(time.RFC1123)) + time.Sleep(waitDuration) + } else { + log.Printf("Maintenance time %s has already passed. Checking now...\n", nextTime.Format(time.RFC1123)) + } + + log.Printf("[%s] Maintenance period reached (+1m). Waiting for fresh node...\n", time.Now().UTC().Format(time.RFC3339)) + if err := waitForFreshNode(tronNodes); err != nil { + log.Printf("Skipping this maintenance report: %v\n", err) + time.Sleep(2 * time.Minute) + continue + } + + log.Printf("[%s] Fetching SR list...\n", time.Now().UTC().Format(time.RFC3339)) + witnesses, err := getWitnessListWithRetry(tronNodes) + if err != nil { + log.Printf("Skipping this maintenance report: %v\n", err) + } else { + if err := sendToSlack(slackWebhook, tronNodes, witnesses, lastVotes, lastTop27); err != nil { + log.Printf("Error sending to Slack: %v\n", err) + } else { + log.Println("Successfully sent SR list to Slack") + lastTop27 = updateLastStatus(witnesses, lastVotes) + saveMonitorState(lastVotes, lastTop27) + } + } + + // Wait a bit before checking for the NEXT maintenance time to avoid double-triggering + time.Sleep(2 * time.Minute) + } +}