perf(center): serve edge target sync from the center TargetCache

center/router/router_funcs.go

@@ -31,6 +31,17 @@ func (rt *Router) statistic(c *gin.Context) {
 	case "recording_rule":
 		model = models.RecordingRule{}
 	case "target":
+		// Answer from the center's TargetCache rather than running
+		// count(*)/max(update_at) over the target table for every edge poll.
+		// It is only a change signal: with several centers the edge's next
+		// /v1/n9e/targets call may hit a center whose cache is at another
+		// stat, so the edge stores the stat derived from the list it actually
+		// received (see memsto TargetCacheType.syncTargets).
+		if rt.TargetCache != nil {
+			total, lastUpdated := rt.TargetCache.Stat()
+			ginx.NewRender(c).Data(&models.Statistics{Total: total, LastUpdated: lastUpdated}, nil)
+			return
+		}
 		model = models.Target{}
 	case "user":
 		model = models.User{}

center/router/router_target.go

@@ -234,9 +234,19 @@ func (rt *Router) targetExtendInfoByIdent(c *gin.Context) {
 	}, nil)
 }
 
+// targetGetsByService is polled by every edge/alert instance whenever the target
+// statistic changes. Serving it from the center's TargetCache keeps those full
+// reloads (target + target_busi_group full scans) off the DB; the cache holds the
+// same rows TargetGetsAll returns, refreshed on the same statistic.
 func (rt *Router) targetGetsByService(c *gin.Context) {
-	lst, err := models.TargetGetsAll(rt.Ctx)
-	ginx.NewRender(c).Data(lst, err)
+	if rt.TargetCache == nil {
+		lst, err := models.TargetGetsAll(rt.Ctx)
+		ginx.NewRender(c).Data(lst, err)
+		return
+	}
+
+	lst, _, _ := rt.TargetCache.Snapshot()
+	ginx.NewRender(c).Data(lst, nil)
 }
 
 func (rt *Router) targetGetTags(c *gin.Context) {

center/router/router_target_service_test.go

@@ -0,0 +1,103 @@
+package router
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sort"
+	"testing"
+
+	"github.com/ccfos/nightingale/v6/memsto"
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/gin-gonic/gin"
+)
+
+// TestTargetServiceEndpointsFromCache covers /v1/n9e/targets and
+// /v1/n9e/statistic?name=target being answered from the center's TargetCache:
+// edges poll both, so neither may touch the DB, and the stat must be the one
+// the served targets were loaded at. Router has no Ctx here, so any DB access
+// would panic.
+func TestTargetServiceEndpointsFromCache(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	tc := &memsto.TargetCacheType{}
+	tc.Set(map[string]*models.Target{
+		"host-1": {Ident: "host-1", HostIp: "10.0.0.1", GroupIds: []int64{1}, TagsJSON: []string{"env=prod"}},
+		"host-2": {Ident: "host-2", HostIp: "10.0.0.2"},
+	}, 2, 1700000000)
+	rt := &Router{TargetCache: tc}
+
+	t.Run("statistic", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		c, _ := gin.CreateTestContext(w)
+		c.Request = httptest.NewRequest(http.MethodGet, "/v1/n9e/statistic?name=target", nil)
+
+		rt.statistic(c)
+
+		var resp struct {
+			Dat *models.Statistics `json:"dat"`
+			Err string             `json:"err"`
+		}
+		decodeServiceResp(t, w, &resp)
+		if resp.Err != "" || resp.Dat == nil {
+			t.Fatalf("err = %q, dat = %v", resp.Err, resp.Dat)
+		}
+		if resp.Dat.Total != 2 || resp.Dat.LastUpdated != 1700000000 {
+			t.Fatalf("stat = %+v, want {2 1700000000}", *resp.Dat)
+		}
+	})
+
+	t.Run("targets", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		c, _ := gin.CreateTestContext(w)
+		c.Request = httptest.NewRequest(http.MethodGet, "/v1/n9e/targets", nil)
+
+		rt.targetGetsByService(c)
+
+		var resp struct {
+			Dat []*models.Target `json:"dat"`
+			Err string           `json:"err"`
+		}
+		decodeServiceResp(t, w, &resp)
+		if resp.Err != "" {
+			t.Fatalf("err = %q", resp.Err)
+		}
+		sort.Slice(resp.Dat, func(i, j int) bool { return resp.Dat[i].Ident < resp.Dat[j].Ident })
+		if len(resp.Dat) != 2 || resp.Dat[0].Ident != "host-1" || resp.Dat[1].Ident != "host-2" {
+			t.Fatalf("targets = %+v, want host-1, host-2", resp.Dat)
+		}
+		if got := resp.Dat[0]; got.HostIp != "10.0.0.1" || len(got.GroupIds) != 1 || got.GroupIds[0] != 1 ||
+			len(got.TagsJSON) != 1 || got.TagsJSON[0] != "env=prod" {
+			t.Fatalf("host-1 = %+v, fields not carried over", got)
+		}
+	})
+}
+
+// TestTargetCacheSnapshotIsolated makes sure Snapshot hands out copies: the
+// cache keeps mutating its own objects (BeatTime/meta refresh) while a caller
+// encodes the snapshot, so they must not share the struct.
+func TestTargetCacheSnapshotIsolated(t *testing.T) {
+	tc := &memsto.TargetCacheType{}
+	tc.Set(map[string]*models.Target{"host-1": {Ident: "host-1", BeatTime: 1}}, 1, 10)
+
+	lst, total, lastUpdated := tc.Snapshot()
+	if len(lst) != 1 || total != 1 || lastUpdated != 10 {
+		t.Fatalf("snapshot = %d targets, stat {%d %d}", len(lst), total, lastUpdated)
+	}
+	lst[0].BeatTime = 99
+
+	cached, _ := tc.Get("host-1")
+	if cached.BeatTime != 1 {
+		t.Fatalf("cached BeatTime = %d, snapshot write leaked into cache", cached.BeatTime)
+	}
+}
+
+func decodeServiceResp(t *testing.T, w *httptest.ResponseRecorder, v interface{}) {
+	t.Helper()
+	if w.Code != http.StatusOK {
+		t.Fatalf("status = %d, body = %s", w.Code, w.Body.String())
+	}
+	if err := json.Unmarshal(w.Body.Bytes(), v); err != nil {
+		t.Fatalf("unmarshal body %s: %v", w.Body.String(), err)
+	}
+}

memsto/target_cache.go

@@ -74,14 +74,38 @@ func (tc *TargetCacheType) Set(m map[string]*models.Target, total, lastUpdated i
 		idx[target.HostIp] = append(idx[target.HostIp], ident)
 	}
 
+	// stat is written under the lock together with targets so that Snapshot/Stat
+	// readers on other goroutines always see a matching (targets, stat) pair.
 	tc.Lock()
 	tc.targets = m
 	tc.targetsIndex = idx
-	tc.Unlock()
-
-	// only one goroutine used, so no need lock
 	tc.statTotal = total
 	tc.statLastUpdated = lastUpdated
+	tc.Unlock()
+}
+
+// Stat returns the (count, max(update_at)) of the target table as of the last
+// full load, i.e. the statistics the cached targets correspond to.
+func (tc *TargetCacheType) Stat() (total, lastUpdated int64) {
+	tc.RLock()
+	defer tc.RUnlock()
+	return tc.statTotal, tc.statLastUpdated
+}
+
+// Snapshot returns shallow copies of all cached targets together with the
+// statistics they were loaded at. Copies are taken under the read lock, so the
+// caller may read (e.g. JSON-encode) them while refreshBeatTime/refreshHostMetas
+// keep updating the cached objects. Slice and map fields are shared and must be
+// treated as read-only.
+func (tc *TargetCacheType) Snapshot() (lst []*models.Target, total, lastUpdated int64) {
+	tc.RLock()
+	defer tc.RUnlock()
+	lst = make([]*models.Target, 0, len(tc.targets))
+	for _, target := range tc.targets {
+		t := *target
+		lst = append(lst, &t)
+	}
+	return lst, tc.statTotal, tc.statLastUpdated
 }
 
 func (tc *TargetCacheType) Get(ident string) (*models.Target, bool) {
@@ -298,7 +322,16 @@ func (tc *TargetCacheType) syncTargets() error {
 		m[lst[i].Ident] = lst[i]
 	}
 
-	tc.Set(m, stat.Total, stat.LastUpdated)
+	total, lastUpdated := stat.Total, stat.LastUpdated
+	if !tc.ctx.IsCenter {
+		// The stat and the list are two requests that may be answered by
+		// different centers, each serving its own TargetCache. Storing the
+		// list under the other center's stat could latch a stale list until
+		// the next target change, so derive the stat from the list itself:
+		// a list older than the centers' stat is then simply reloaded.
+		total, lastUpdated = targetsStat(lst)
+	}
+	tc.Set(m, total, lastUpdated)
 	tc.metaSyncCycle = 0 // 全量同步已包含 meta，重置计数器
 
 	ms := time.Since(start).Milliseconds()
@@ -309,6 +342,17 @@ func (tc *TargetCacheType) syncTargets() error {
 	return nil
 }
 
+// targetsStat computes the (count(*), max(update_at)) statistic that
+// TargetStatistics would report for exactly these targets.
+func targetsStat(lst []*models.Target) (total, lastUpdated int64) {
+	for _, t := range lst {
+		if t.UpdateAt > lastUpdated {
+			lastUpdated = t.UpdateAt
+		}
+	}
+	return int64(len(lst)), lastUpdated
+}
+
 // get host update time
 func (tc *TargetCacheType) GetHostUpdateTime(targets []string) map[string]int64 {
 	if tc.redis == nil {

memsto/target_cache_test.go

@@ -0,0 +1,68 @@
+package memsto
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/ccfos/nightingale/v6/conf"
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/ctx"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// TestSyncTargetsEdgeStoresListStat covers an edge whose stat and list
+// requests are answered by two centers at different cache states: the stat
+// comes from a center that has already reloaded, the list from one that has
+// not. The edge must store the stat of the list it received, so that the next
+// poll sees a change and reloads instead of latching the stale list.
+func TestSyncTargetsEdgeStoresListStat(t *testing.T) {
+	stale := []*models.Target{{Ident: "host-1", UpdateAt: 100}}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var dat interface{}
+		switch r.URL.Path {
+		case "/v1/n9e/statistic":
+			dat = &models.Statistics{Total: 2, LastUpdated: 300}
+		case "/v1/n9e/targets":
+			dat = stale
+		default:
+			http.NotFound(w, r)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(map[string]interface{}{"dat": dat, "err": ""})
+	}))
+	defer srv.Close()
+
+	tc := &TargetCacheType{
+		statTotal:       -1,
+		statLastUpdated: -1,
+		ctx:             &ctx.Context{IsCenter: false, CenterApi: conf.CenterApi{Addrs: []string{srv.URL}}},
+		stats: &Stats{
+			GaugeCronDuration: prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_cron_duration"}, []string{"name"}),
+			GaugeSyncNumber:   prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "test_sync_number"}, []string{"name"}),
+		},
+		targets:      make(map[string]*models.Target),
+		targetsIndex: make(map[string][]string),
+	}
+
+	if err := tc.syncTargets(); err != nil {
+		t.Fatalf("syncTargets: %v", err)
+	}
+	if total, last := tc.Stat(); total != 1 || last != 100 {
+		t.Fatalf("stored stat = {%d %d}, want {1 100} derived from the received list", total, last)
+	}
+	if !tc.StatChanged(2, 300) {
+		t.Fatal("StatChanged(2, 300) = false: the stale list would never be reloaded")
+	}
+}
+
+func TestTargetsStat(t *testing.T) {
+	if total, last := targetsStat(nil); total != 0 || last != 0 {
+		t.Fatalf("empty = {%d %d}, want {0 0}", total, last)
+	}
+	total, last := targetsStat([]*models.Target{{UpdateAt: 5}, {UpdateAt: 9}, {UpdateAt: 7}})
+	if total != 3 || last != 9 {
+		t.Fatalf("stat = {%d %d}, want {3 9}", total, last)
+	}
+}

models/target_busi_group.go

@@ -132,8 +132,9 @@ func TargetDeleteBgids(tx *gorm.DB, idents []string) error {
 func TargetOverrideBgids(ctx *ctx.Context, idents []string, bgids []int64, tags []string) error {
 	return DB(ctx).Transaction(func(tx *gorm.DB) error {
 		// 先删除旧的关联
-		if err := tx.Where("target_ident IN ?", idents).Delete(&TargetBusiGroup{}).Error; err != nil {
-			return err
+		del := tx.Where("target_ident IN ?", idents).Delete(&TargetBusiGroup{})
+		if del.Error != nil {
+			return del.Error
 		}
 
 		// 准备新的关联数据
@@ -151,7 +152,15 @@ func TargetOverrideBgids(ctx *ctx.Context, idents []string, bgids []int64, tags
 		}
 
 		if len(lst) == 0 {
-			return nil
+			// Clearing all groups changes nothing in the target table itself, so
+			// bump update_at for syncTargets to refresh GroupIds. Only when rows
+			// were actually removed: heartbeats with overwrite_gids and an empty
+			// gid call this every time, and an unconditional bump would force a
+			// full target reload on every center and edge each cycle.
+			if del.RowsAffected == 0 {
+				return nil
+			}
+			return tx.Model(&Target{}).Where("ident IN ?", idents).Update("update_at", updateAt).Error
 		}
 
 		// 添加新的关联

models/target_busi_group_test.go

@@ -0,0 +1,74 @@
+package models_test
+
+import (
+	"testing"
+
+	"github.com/glebarez/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+
+	"github.com/ccfos/nightingale/v6/models"
+	"github.com/ccfos/nightingale/v6/pkg/ctx"
+)
+
+// TestTargetOverrideBgidsEmptyBumpsUpdateAt covers resetting a target to no
+// busi group: the target row itself is unchanged, so update_at must be bumped
+// for TargetCache (and everything synced from it) to drop the old GroupIds.
+// Repeating the reset (heartbeats with overwrite_gids and an empty gid) must
+// not bump it again, or every cycle would force a full target reload.
+func TestTargetOverrideBgidsEmptyBumpsUpdateAt(t *testing.T) {
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+	})
+	if err != nil {
+		t.Fatalf("open sqlite: %v", err)
+	}
+	sqlDB, err := db.DB()
+	if err != nil {
+		t.Fatalf("sql db: %v", err)
+	}
+	sqlDB.SetMaxOpenConns(1) // keep the single in-memory database across transactions
+	if err := db.AutoMigrate(&models.Target{}, &models.TargetBusiGroup{}); err != nil {
+		t.Fatalf("migrate: %v", err)
+	}
+	c := &ctx.Context{DB: db, IsCenter: true}
+
+	if err := db.Create(&models.Target{Ident: "host-1", UpdateAt: 100}).Error; err != nil {
+		t.Fatalf("create target: %v", err)
+	}
+	if err := db.Create(&models.TargetBusiGroup{TargetIdent: "host-1", GroupId: 1, UpdateAt: 100}).Error; err != nil {
+		t.Fatalf("create binding: %v", err)
+	}
+
+	updateAt := func() int64 {
+		var tgt models.Target
+		if err := db.Where("ident = ?", "host-1").First(&tgt).Error; err != nil {
+			t.Fatalf("get target: %v", err)
+		}
+		return tgt.UpdateAt
+	}
+
+	if err := models.TargetOverrideBgids(c, []string{"host-1"}, nil, nil); err != nil {
+		t.Fatalf("override to empty: %v", err)
+	}
+	var bindings int64
+	db.Model(&models.TargetBusiGroup{}).Where("target_ident = ?", "host-1").Count(&bindings)
+	if bindings != 0 {
+		t.Fatalf("bindings = %d, want 0", bindings)
+	}
+	bumped := updateAt()
+	if bumped <= 100 {
+		t.Fatalf("update_at = %d, want bumped past 100 after clearing groups", bumped)
+	}
+
+	// Already ungrouped: nothing changes, so update_at must stay put.
+	if err := db.Model(&models.Target{}).Where("ident = ?", "host-1").Update("update_at", 200).Error; err != nil {
+		t.Fatalf("reset update_at: %v", err)
+	}
+	if err := models.TargetOverrideBgids(c, []string{"host-1"}, nil, nil); err != nil {
+		t.Fatalf("override to empty again: %v", err)
+	}
+	if got := updateAt(); got != 200 {
+		t.Fatalf("update_at = %d, want 200 unchanged when no binding was removed", got)
+	}
+}