From 8ddb46c82209dcd4877175a4c494b5391f2f2ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9F=B3=E8=BF=AA=E6=BD=87?= <41122304+Dixiao-L@users.noreply.github.com> Date: Tue, 10 Nov 2026 05:00:04 +0800 Subject: [PATCH] fix: publish collected statistics after status transactions commit (#771) * fix: publish collected statistics after status transactions commit Release the SQLite connection before publishing to bounded subscribers that may read peer ownership. Schedule metrics and publish connection- state changes only after a successful status update. Signed-off-by: Dixiao-L * ensure that tests are successful against the current master branch --------- Signed-off-by: Dixiao-L Co-authored-by: Christoph --- internal/app/wireguard/statistics.go | 32 +-- .../app/wireguard/statistics_commit_test.go | 225 ++++++++++++++++++ .../wireguard/statistics_websocket_test.go | 161 +++++++++++++ 3 files changed, 403 insertions(+), 15 deletions(-) create mode 100644 internal/app/wireguard/statistics_commit_test.go create mode 100644 internal/app/wireguard/statistics_websocket_test.go diff --git a/internal/app/wireguard/statistics.go b/internal/app/wireguard/statistics.go index f16db52f..69b78100 100644 --- a/internal/app/wireguard/statistics.go +++ b/internal/app/wireguard/statistics.go @@ -123,9 +123,11 @@ func (c *StatisticsCollector) collectInterfaceData(ctx context.Context) { continue } now := time.Now() + var td domain.TrafficDelta + var committedStatus domain.InterfaceStatus err = c.db.UpdateInterfaceStatus(ctx, in.Identifier, func(i *domain.InterfaceStatus) (*domain.InterfaceStatus, error) { - td := domain.CalculateTrafficDelta( + td = domain.CalculateTrafficDelta( string(in.Identifier), i.UpdatedAt, now, i.BytesTransmitted, physicalInterface.BytesUpload, @@ -135,17 +137,18 @@ func (c *StatisticsCollector) collectInterfaceData(ctx context.Context) { i.BytesReceived = physicalInterface.BytesDownload i.BytesTransmitted = physicalInterface.BytesUpload - // Update prometheus metrics - go c.updateInterfaceMetrics(*i) - - // Publish stats update event - c.bus.Publish(app.TopicInterfaceStatsUpdated, td) + committedStatus = *i return i, nil }) if err != nil { slog.Warn("failed to update interface status", "interface", in.Identifier, "error", err) + continue } + // Subscribers may access SQLite or block on a slow websocket. Never + // publish while the status transaction owns the only DB connection. + go c.updateInterfaceMetrics(committedStatus) + c.bus.Publish(app.TopicInterfaceStatsUpdated, td) slog.Debug("updated interface status", "interface", in.Identifier) } } @@ -187,6 +190,7 @@ func (c *StatisticsCollector) collectPeerData(ctx context.Context) { for _, peer := range peers { var connectionStateChanged bool var newPeerStatus domain.PeerStatus + var td domain.TrafficDelta err = c.db.UpdatePeerStatus(ctx, peer.Identifier, func(p *domain.PeerStatus) (*domain.PeerStatus, error) { wasConnected := p.IsConnected @@ -196,7 +200,7 @@ func (c *StatisticsCollector) collectPeerData(ctx context.Context) { lastHandshake = &peer.LastHandshake } - td := domain.CalculateTrafficDelta( + td = domain.CalculateTrafficDelta( string(peer.Identifier), p.UpdatedAt, now, p.BytesTransmitted, peer.BytesDownload, @@ -217,22 +221,20 @@ func (c *StatisticsCollector) collectPeerData(ctx context.Context) { slog.Debug("peer connection state changed", "peer", peer.Identifier, "connected", p.IsConnected) connectionStateChanged = true - newPeerStatus = *p // store new status for event publishing } - // Update prometheus metrics - go c.updatePeerMetrics(ctx, *p) - - // Publish stats update event - c.bus.Publish(app.TopicPeerStatsUpdated, td) + newPeerStatus = *p return p, nil }) if err != nil { slog.Warn("failed to update peer status", "peer", peer.Identifier, "error", err) - } else { - slog.Debug("updated peer status", "peer", peer.Identifier) + continue } + // Release the SQLite transaction before notifying subscribers. + go c.updatePeerMetrics(ctx, newPeerStatus) + c.bus.Publish(app.TopicPeerStatsUpdated, td) + slog.Debug("updated peer status", "peer", peer.Identifier) if connectionStateChanged { peerModel, err := c.db.GetPeer(ctx, peer.Identifier) diff --git a/internal/app/wireguard/statistics_commit_test.go b/internal/app/wireguard/statistics_commit_test.go new file mode 100644 index 00000000..a7b74d78 --- /dev/null +++ b/internal/app/wireguard/statistics_commit_test.go @@ -0,0 +1,225 @@ +package wireguard + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/h44z/wg-portal/internal/adapters" + "github.com/h44z/wg-portal/internal/app" + "github.com/h44z/wg-portal/internal/config" + "github.com/h44z/wg-portal/internal/domain" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "gorm.io/gorm/schema" +) + +// Only return interfaces once, so each test observes exactly one collection pass. +type statisticsTestRepo struct { + *adapters.SqlRepo + once sync.Once +} + +func (r *statisticsTestRepo) GetAllInterfaces(ctx context.Context) (interfaces []domain.Interface, err error) { + r.once.Do(func() { interfaces, err = r.SqlRepo.GetAllInterfaces(ctx) }) + return +} + +type statisticsTestController struct{ domain.InterfaceController } + +func (*statisticsTestController) GetInterface(_ context.Context, id domain.InterfaceIdentifier) (*domain.PhysicalInterface, error) { + return &domain.PhysicalInterface{Identifier: id, BytesUpload: 42, BytesDownload: 24}, nil +} +func (*statisticsTestController) GetPeers(context.Context, domain.InterfaceIdentifier) ([]domain.PhysicalPeer, error) { + return []domain.PhysicalPeer{{Identifier: "peer", BytesUpload: 42, BytesDownload: 24, LastHandshake: time.Now()}}, nil +} + +type statisticsTestBus struct{ publish func(string, ...any) } + +func (b statisticsTestBus) Publish(topic string, args ...any) { b.publish(topic, args...) } +func (statisticsTestBus) Subscribe(string, any) error { return nil } + +type statisticsTestMetrics struct{ statuses chan any } + +func (m statisticsTestMetrics) UpdateInterfaceMetrics(s domain.InterfaceStatus) { m.statuses <- s } +func (m statisticsTestMetrics) UpdatePeerMetrics(_ *domain.Peer, s domain.PeerStatus) { + m.statuses <- s +} +func (statisticsTestMetrics) DeletePeerMetrics(domain.Peer) {} + +func newStatisticsTestCollector(t *testing.T) (*StatisticsCollector, *gorm.DB, statisticsTestMetrics) { + t.Helper() + schema.RegisterSerializer("encstr", app.NewGormEncryptedStringSerializer("")) + cfg := &config.Config{} + cfg.Database = config.DatabaseConfig{Type: config.DatabaseSQLite, DSN: filepath.Join(t.TempDir(), "statistics.db")} + cfg.Statistics.DataCollectionInterval = time.Millisecond + cfg.Backend.ReKeyTimeoutInterval = time.Minute + db, err := adapters.NewDatabase(cfg.Database) + require.NoError(t, err) + sqlDB, err := db.DB() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + require.Equal(t, 1, sqlDB.Stats().MaxOpenConnections) + repo, err := adapters.NewSqlRepository(db, cfg) + require.NoError(t, err) + require.NoError(t, db.Create(&domain.Interface{Identifier: "wg0"}).Error) + require.NoError(t, db.Create(&domain.User{Identifier: "user"}).Error) + require.NoError(t, db.Create(&domain.Peer{Identifier: "peer", UserIdentifier: "user", InterfaceIdentifier: "wg0"}).Error) + old := time.Now().Add(-time.Minute) + require.NoError(t, db.Create(&domain.PeerStatus{PeerId: "peer", UpdatedAt: old, BytesReceived: 2, BytesTransmitted: 1}).Error) + require.NoError(t, db.Create(&domain.InterfaceStatus{InterfaceId: "wg0", UpdatedAt: old, BytesReceived: 2, BytesTransmitted: 1}).Error) + metrics := statisticsTestMetrics{statuses: make(chan any, 4)} + collector := &StatisticsCollector{ + cfg: cfg, db: &statisticsTestRepo{SqlRepo: repo}, ms: metrics, + wg: &ControllerManager{controllers: map[domain.InterfaceBackend]backendInstance{ + config.LocalBackendName: {Implementation: &statisticsTestController{}}, + }}, + } + return collector, db, metrics +} + +func startStatisticsTestCollector(t *testing.T, collector *StatisticsCollector, kind string) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + if kind == "peer" { + collector.collectPeerData(ctx) + } else { + collector.collectInterfaceData(ctx) + } + }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Error("collector did not stop") + } + }) +} + +func receiveStatisticsTest[T any](t *testing.T, ch <-chan T) T { + t.Helper() + select { + case value := <-ch: + return value + case <-time.After(3 * time.Second): + t.Fatal("timed out waiting for statistics collection") + } + var zero T + return zero +} + +// The subscriber reads through the real one-connection pool before Publish returns. +// Publishing inside the transaction cannot satisfy this read. +func TestStatisticsSubscribersReadCommittedStatus(t *testing.T) { + for _, kind := range []string{"peer", "interface"} { + t.Run(kind, func(t *testing.T) { + collector, db, metrics := newStatisticsTestCollector(t) + results := make(chan error, 1) + state := make(chan domain.PeerStatus, 1) + collector.bus = statisticsTestBus{publish: func(topic string, args ...any) { + if topic == app.TopicPeerStateChanged { + state <- args[0].(domain.PeerStatus) + return + } + readCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if kind == "peer" { + var status domain.PeerStatus + err := db.WithContext(readCtx).First(&status, "identifier = ?", "peer").Error + if err == nil && (status.BytesReceived != 42 || status.BytesTransmitted != 24 || !status.IsConnected) { + err = errors.New("peer event preceded committed status") + } + results <- err + } else { + var status domain.InterfaceStatus + err := db.WithContext(readCtx).First(&status, "identifier = ?", "wg0").Error + if err == nil && (status.BytesReceived != 24 || status.BytesTransmitted != 42) { + err = errors.New("interface event preceded committed status") + } + results <- err + } + }} + startStatisticsTestCollector(t, collector, kind) + require.NoError(t, receiveStatisticsTest(t, results)) + status := receiveStatisticsTest(t, metrics.statuses) + if kind == "peer" { + s := status.(domain.PeerStatus) + require.EqualValues(t, 42, s.BytesReceived) + require.EqualValues(t, 24, s.BytesTransmitted) + require.True(t, s.IsConnected) + require.Equal(t, s, receiveStatisticsTest(t, state)) + } else { + s := status.(domain.InterfaceStatus) + require.EqualValues(t, 24, s.BytesReceived) + require.EqualValues(t, 42, s.BytesTransmitted) + } + }) + } +} + +// Fail at the COMMIT boundary after the callback and real SQLite save succeed. +// Roll back the underlying transaction so the injected failure releases its connection. +type statisticsCommitFailurePool struct{ *sql.DB } + +func (p statisticsCommitFailurePool) BeginTx(ctx context.Context, options *sql.TxOptions) (gorm.ConnPool, error) { + tx, err := p.DB.BeginTx(ctx, options) + if err != nil { + return nil, err + } + return &statisticsCommitFailureTx{Tx: tx}, nil +} + +type statisticsCommitFailureTx struct{ *sql.Tx } + +func (tx statisticsCommitFailureTx) Commit() error { + return errors.Join(errors.New("injected commit failure"), tx.Tx.Rollback()) +} + +func TestStatisticsRollbackDoesNotPublish(t *testing.T) { + for _, kind := range []string{"peer", "interface"} { + for _, failure := range []string{"save", "commit"} { + t.Run(kind+"/"+failure, func(t *testing.T) { + collector, db, metrics := newStatisticsTestCollector(t) + if failure == "commit" { + pool, err := db.DB() + require.NoError(t, err) + db.Statement.ConnPool = &statisticsCommitFailurePool{DB: pool} + } + failed := make(chan struct{}, 1) + require.NoError(t, db.Callback().Update().Before("gorm:update").Register("test:reject-status", func(tx *gorm.DB) { + if failure == "save" { + tx.AddError(errors.New("injected status write failure")) + } + failed <- struct{}{} + })) + published := make(chan string, 4) + collector.bus = statisticsTestBus{publish: func(topic string, _ ...any) { published <- topic }} + startStatisticsTestCollector(t, collector, kind) + receiveStatisticsTest(t, failed) + // This read also waits for rollback to release the only connection. + var peer domain.PeerStatus + var iface domain.InterfaceStatus + require.NoError(t, db.First(&peer, "identifier = ?", "peer").Error) + require.NoError(t, db.First(&iface, "identifier = ?", "wg0").Error) + require.EqualValues(t, 2, peer.BytesReceived) + require.False(t, peer.IsConnected) + require.EqualValues(t, 1, iface.BytesTransmitted) + select { + case topic := <-published: + t.Errorf("rolled back update published %s", topic) + case status := <-metrics.statuses: + t.Errorf("rolled back update scheduled metrics: %+v", status) + case <-time.After(100 * time.Millisecond): + } + }) + } + } +} diff --git a/internal/app/wireguard/statistics_websocket_test.go b/internal/app/wireguard/statistics_websocket_test.go new file mode 100644 index 00000000..2c0e825f --- /dev/null +++ b/internal/app/wireguard/statistics_websocket_test.go @@ -0,0 +1,161 @@ +package wireguard + +import ( + "context" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/go-pkgz/routegroup" + "github.com/gorilla/websocket" + "github.com/h44z/wg-portal/internal/adapters" + "github.com/h44z/wg-portal/internal/app" + "github.com/h44z/wg-portal/internal/app/api/v0/handlers" + "github.com/h44z/wg-portal/internal/domain" + "github.com/stretchr/testify/require" + evbus "github.com/vardius/message-bus" +) + +// Only authentication is replaced; the real endpoint performs the ownership +// check, SQLite query, caching, and WebSocket writes for a non-admin user. +type statisticsWebsocketAuth struct{ handlers.Authenticator } + +func (statisticsWebsocketAuth) LoggedIn(...handlers.Scope) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r.WithContext(domain.SetUserInfo(r.Context(), &domain.ContextUserInfo{Id: "user"}))) + }) + } +} + +func (statisticsWebsocketAuth) RequireNonDefaultPassword() func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return next + } +} + +type statisticsWebsocketBus struct { + evbus.MessageBus + ready chan struct{} +} + +func (b statisticsWebsocketBus) Subscribe(topic string, fn any) error { + err := b.MessageBus.Subscribe(topic, fn) + // The endpoint subscribes to peer stats before interface stats. + if err == nil && topic == app.TopicInterfaceStatsUpdated { + close(b.ready) + } + return err +} + +type statisticsOwnershipService struct { + *adapters.SqlRepo + entered chan struct{} + release chan struct{} + result chan error + once *sync.Once +} + +func (s statisticsOwnershipService) GetPeer(ctx context.Context, id domain.PeerIdentifier) (*domain.Peer, error) { + s.once.Do(func() { close(s.entered) }) + select { + case <-s.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + // Bound the old deadlock as well, so a failing regression cleans up all workers. + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + peer, err := s.SqlRepo.GetPeer(ctx, id) + s.result <- err + return peer, err +} + +type statisticsBackpressureRepo struct { + *statisticsTestRepo + bus evbus.MessageBus + ownership statisticsOwnershipService +} + +func (r statisticsBackpressureRepo) UpdatePeerStatus(ctx context.Context, id domain.PeerIdentifier, + update func(*domain.PeerStatus) (*domain.PeerStatus, error)) error { + return r.SqlRepo.UpdatePeerStatus(ctx, id, func(status *domain.PeerStatus) (*domain.PeerStatus, error) { + // The transaction now owns SQLite's only connection. Start the real + // ownership handler, park it, then fill its one-slot subscriber queue. + event := domain.TrafficDelta{EntityId: string(id)} + r.bus.Publish(app.TopicPeerStatsUpdated, event) + select { + case <-r.ownership.entered: + case <-ctx.Done(): + return nil, ctx.Err() + } + r.bus.Publish(app.TopicPeerStatsUpdated, event) + close(r.ownership.release) + // The collector's third Publish must happen after this transaction returns. + return update(status) + }) +} + +func TestStatisticsBoundedBusWithNonAdminWebsocket(t *testing.T) { + collector, db, metrics := newStatisticsTestCollector(t) + repo := collector.db.(*statisticsTestRepo) + bus := statisticsWebsocketBus{MessageBus: evbus.New(1), ready: make(chan struct{})} + ownership := statisticsOwnershipService{SqlRepo: repo.SqlRepo, + entered: make(chan struct{}), release: make(chan struct{}), result: make(chan error, 4), once: &sync.Once{}} + collector.db = statisticsBackpressureRepo{statisticsTestRepo: repo, bus: bus, ownership: ownership} + collector.bus = bus + + state := make(chan domain.PeerStatus, 1) + require.NoError(t, bus.Subscribe(app.TopicPeerStateChanged, func(status domain.PeerStatus, _ domain.Peer) { state <- status })) + defer bus.Close(app.TopicPeerStateChanged) + mux := http.NewServeMux() + endpointDone := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mux.ServeHTTP(w, r) + close(endpointDone) + })) + t.Cleanup(server.Close) + collector.cfg.Web.ExternalUrl = server.URL + endpoint := handlers.NewWebsocketEndpoint(collector.cfg, statisticsWebsocketAuth{}, bus, ownership) + endpoint.RegisterRoutes(routegroup.New(mux)) + conn, _, err := websocket.DefaultDialer.Dial("ws"+server.URL[len("http"):]+"/ws", + http.Header{"Origin": []string{server.URL}}) + require.NoError(t, err) + t.Cleanup(func() { conn.Close(); receiveStatisticsTest(t, endpointDone) }) + receiveStatisticsTest(t, bus.ready) + + // A separate reader models login/API traffic competing for the same pool. + readResult := make(chan error, 1) + var readers sync.WaitGroup + readers.Add(1) + go func() { + defer readers.Done() + <-ownership.entered + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + var user domain.User + readResult <- db.WithContext(ctx).First(&user, "identifier = ?", "user").Error + }() + t.Cleanup(readers.Wait) + startStatisticsTestCollector(t, collector, "peer") + require.NoError(t, receiveStatisticsTest(t, readResult), "concurrent API query must not be starved by a blocked Publish") + require.NoError(t, receiveStatisticsTest(t, ownership.result), "ownership query must not wait on the publishing transaction") + + require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second))) + for range 3 { + var message struct { + Type string `json:"type"` + Data domain.TrafficDelta `json:"data"` + } + require.NoError(t, conn.ReadJSON(&message)) + require.Equal(t, "peer_stats", message.Type) + require.Equal(t, "peer", message.Data.EntityId) + } + require.True(t, receiveStatisticsTest(t, state).IsConnected) + receiveStatisticsTest(t, metrics.statuses) + var status domain.PeerStatus + require.NoError(t, db.First(&status, "identifier = ?", "peer").Error) + require.EqualValues(t, 42, status.BytesReceived) +}