fix: publish collected statistics after status transactions commit (#771)

* fix: publish collected statistics after status transactions commit

Release the SQLite connection before publishing to bounded subscribers
that may read peer ownership. Schedule metrics and publish connection-
state changes only after a successful status update.

Signed-off-by: Dixiao-L <stgowl@outlook.com>

* ensure that tests are successful against the current master branch

---------

Signed-off-by: Dixiao-L <stgowl@outlook.com>
Co-authored-by: Christoph <christoph.h@sprinternet.at>
This commit is contained in:
柳迪潇
2026-11-09 22:00:04 +01:00
committed by GitHub
co-authored by Christoph
parent 440a619aea
commit 8ddb46c822
3 changed files with 403 additions and 15 deletions
+17 -15
View File
@@ -123,9 +123,11 @@ func (c *StatisticsCollector) collectInterfaceData(ctx context.Context) {
continue
}
now := time.Now()
var td domain.TrafficDelta
var committedStatus domain.InterfaceStatus
err = c.db.UpdateInterfaceStatus(ctx, in.Identifier,
func(i *domain.InterfaceStatus) (*domain.InterfaceStatus, error) {
td := domain.CalculateTrafficDelta(
td = domain.CalculateTrafficDelta(
string(in.Identifier),
i.UpdatedAt, now,
i.BytesTransmitted, physicalInterface.BytesUpload,
@@ -135,17 +137,18 @@ func (c *StatisticsCollector) collectInterfaceData(ctx context.Context) {
i.BytesReceived = physicalInterface.BytesDownload
i.BytesTransmitted = physicalInterface.BytesUpload
// Update prometheus metrics
go c.updateInterfaceMetrics(*i)
// Publish stats update event
c.bus.Publish(app.TopicInterfaceStatsUpdated, td)
committedStatus = *i
return i, nil
})
if err != nil {
slog.Warn("failed to update interface status", "interface", in.Identifier, "error", err)
continue
}
// Subscribers may access SQLite or block on a slow websocket. Never
// publish while the status transaction owns the only DB connection.
go c.updateInterfaceMetrics(committedStatus)
c.bus.Publish(app.TopicInterfaceStatsUpdated, td)
slog.Debug("updated interface status", "interface", in.Identifier)
}
}
@@ -187,6 +190,7 @@ func (c *StatisticsCollector) collectPeerData(ctx context.Context) {
for _, peer := range peers {
var connectionStateChanged bool
var newPeerStatus domain.PeerStatus
var td domain.TrafficDelta
err = c.db.UpdatePeerStatus(ctx, peer.Identifier,
func(p *domain.PeerStatus) (*domain.PeerStatus, error) {
wasConnected := p.IsConnected
@@ -196,7 +200,7 @@ func (c *StatisticsCollector) collectPeerData(ctx context.Context) {
lastHandshake = &peer.LastHandshake
}
td := domain.CalculateTrafficDelta(
td = domain.CalculateTrafficDelta(
string(peer.Identifier),
p.UpdatedAt, now,
p.BytesTransmitted, peer.BytesDownload,
@@ -217,22 +221,20 @@ func (c *StatisticsCollector) collectPeerData(ctx context.Context) {
slog.Debug("peer connection state changed",
"peer", peer.Identifier, "connected", p.IsConnected)
connectionStateChanged = true
newPeerStatus = *p // store new status for event publishing
}
// Update prometheus metrics
go c.updatePeerMetrics(ctx, *p)
// Publish stats update event
c.bus.Publish(app.TopicPeerStatsUpdated, td)
newPeerStatus = *p
return p, nil
})
if err != nil {
slog.Warn("failed to update peer status", "peer", peer.Identifier, "error", err)
} else {
slog.Debug("updated peer status", "peer", peer.Identifier)
continue
}
// Release the SQLite transaction before notifying subscribers.
go c.updatePeerMetrics(ctx, newPeerStatus)
c.bus.Publish(app.TopicPeerStatsUpdated, td)
slog.Debug("updated peer status", "peer", peer.Identifier)
if connectionStateChanged {
peerModel, err := c.db.GetPeer(ctx, peer.Identifier)
@@ -0,0 +1,225 @@
package wireguard
import (
"context"
"database/sql"
"errors"
"path/filepath"
"sync"
"testing"
"time"
"github.com/h44z/wg-portal/internal/adapters"
"github.com/h44z/wg-portal/internal/app"
"github.com/h44z/wg-portal/internal/config"
"github.com/h44z/wg-portal/internal/domain"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"gorm.io/gorm/schema"
)
// Only return interfaces once, so each test observes exactly one collection pass.
type statisticsTestRepo struct {
*adapters.SqlRepo
once sync.Once
}
func (r *statisticsTestRepo) GetAllInterfaces(ctx context.Context) (interfaces []domain.Interface, err error) {
r.once.Do(func() { interfaces, err = r.SqlRepo.GetAllInterfaces(ctx) })
return
}
type statisticsTestController struct{ domain.InterfaceController }
func (*statisticsTestController) GetInterface(_ context.Context, id domain.InterfaceIdentifier) (*domain.PhysicalInterface, error) {
return &domain.PhysicalInterface{Identifier: id, BytesUpload: 42, BytesDownload: 24}, nil
}
func (*statisticsTestController) GetPeers(context.Context, domain.InterfaceIdentifier) ([]domain.PhysicalPeer, error) {
return []domain.PhysicalPeer{{Identifier: "peer", BytesUpload: 42, BytesDownload: 24, LastHandshake: time.Now()}}, nil
}
type statisticsTestBus struct{ publish func(string, ...any) }
func (b statisticsTestBus) Publish(topic string, args ...any) { b.publish(topic, args...) }
func (statisticsTestBus) Subscribe(string, any) error { return nil }
type statisticsTestMetrics struct{ statuses chan any }
func (m statisticsTestMetrics) UpdateInterfaceMetrics(s domain.InterfaceStatus) { m.statuses <- s }
func (m statisticsTestMetrics) UpdatePeerMetrics(_ *domain.Peer, s domain.PeerStatus) {
m.statuses <- s
}
func (statisticsTestMetrics) DeletePeerMetrics(domain.Peer) {}
func newStatisticsTestCollector(t *testing.T) (*StatisticsCollector, *gorm.DB, statisticsTestMetrics) {
t.Helper()
schema.RegisterSerializer("encstr", app.NewGormEncryptedStringSerializer(""))
cfg := &config.Config{}
cfg.Database = config.DatabaseConfig{Type: config.DatabaseSQLite, DSN: filepath.Join(t.TempDir(), "statistics.db")}
cfg.Statistics.DataCollectionInterval = time.Millisecond
cfg.Backend.ReKeyTimeoutInterval = time.Minute
db, err := adapters.NewDatabase(cfg.Database)
require.NoError(t, err)
sqlDB, err := db.DB()
require.NoError(t, err)
t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
require.Equal(t, 1, sqlDB.Stats().MaxOpenConnections)
repo, err := adapters.NewSqlRepository(db, cfg)
require.NoError(t, err)
require.NoError(t, db.Create(&domain.Interface{Identifier: "wg0"}).Error)
require.NoError(t, db.Create(&domain.User{Identifier: "user"}).Error)
require.NoError(t, db.Create(&domain.Peer{Identifier: "peer", UserIdentifier: "user", InterfaceIdentifier: "wg0"}).Error)
old := time.Now().Add(-time.Minute)
require.NoError(t, db.Create(&domain.PeerStatus{PeerId: "peer", UpdatedAt: old, BytesReceived: 2, BytesTransmitted: 1}).Error)
require.NoError(t, db.Create(&domain.InterfaceStatus{InterfaceId: "wg0", UpdatedAt: old, BytesReceived: 2, BytesTransmitted: 1}).Error)
metrics := statisticsTestMetrics{statuses: make(chan any, 4)}
collector := &StatisticsCollector{
cfg: cfg, db: &statisticsTestRepo{SqlRepo: repo}, ms: metrics,
wg: &ControllerManager{controllers: map[domain.InterfaceBackend]backendInstance{
config.LocalBackendName: {Implementation: &statisticsTestController{}},
}},
}
return collector, db, metrics
}
func startStatisticsTestCollector(t *testing.T, collector *StatisticsCollector, kind string) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() {
defer close(done)
if kind == "peer" {
collector.collectPeerData(ctx)
} else {
collector.collectInterfaceData(ctx)
}
}()
t.Cleanup(func() {
cancel()
select {
case <-done:
case <-time.After(3 * time.Second):
t.Error("collector did not stop")
}
})
}
func receiveStatisticsTest[T any](t *testing.T, ch <-chan T) T {
t.Helper()
select {
case value := <-ch:
return value
case <-time.After(3 * time.Second):
t.Fatal("timed out waiting for statistics collection")
}
var zero T
return zero
}
// The subscriber reads through the real one-connection pool before Publish returns.
// Publishing inside the transaction cannot satisfy this read.
func TestStatisticsSubscribersReadCommittedStatus(t *testing.T) {
for _, kind := range []string{"peer", "interface"} {
t.Run(kind, func(t *testing.T) {
collector, db, metrics := newStatisticsTestCollector(t)
results := make(chan error, 1)
state := make(chan domain.PeerStatus, 1)
collector.bus = statisticsTestBus{publish: func(topic string, args ...any) {
if topic == app.TopicPeerStateChanged {
state <- args[0].(domain.PeerStatus)
return
}
readCtx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if kind == "peer" {
var status domain.PeerStatus
err := db.WithContext(readCtx).First(&status, "identifier = ?", "peer").Error
if err == nil && (status.BytesReceived != 42 || status.BytesTransmitted != 24 || !status.IsConnected) {
err = errors.New("peer event preceded committed status")
}
results <- err
} else {
var status domain.InterfaceStatus
err := db.WithContext(readCtx).First(&status, "identifier = ?", "wg0").Error
if err == nil && (status.BytesReceived != 24 || status.BytesTransmitted != 42) {
err = errors.New("interface event preceded committed status")
}
results <- err
}
}}
startStatisticsTestCollector(t, collector, kind)
require.NoError(t, receiveStatisticsTest(t, results))
status := receiveStatisticsTest(t, metrics.statuses)
if kind == "peer" {
s := status.(domain.PeerStatus)
require.EqualValues(t, 42, s.BytesReceived)
require.EqualValues(t, 24, s.BytesTransmitted)
require.True(t, s.IsConnected)
require.Equal(t, s, receiveStatisticsTest(t, state))
} else {
s := status.(domain.InterfaceStatus)
require.EqualValues(t, 24, s.BytesReceived)
require.EqualValues(t, 42, s.BytesTransmitted)
}
})
}
}
// Fail at the COMMIT boundary after the callback and real SQLite save succeed.
// Roll back the underlying transaction so the injected failure releases its connection.
type statisticsCommitFailurePool struct{ *sql.DB }
func (p statisticsCommitFailurePool) BeginTx(ctx context.Context, options *sql.TxOptions) (gorm.ConnPool, error) {
tx, err := p.DB.BeginTx(ctx, options)
if err != nil {
return nil, err
}
return &statisticsCommitFailureTx{Tx: tx}, nil
}
type statisticsCommitFailureTx struct{ *sql.Tx }
func (tx statisticsCommitFailureTx) Commit() error {
return errors.Join(errors.New("injected commit failure"), tx.Tx.Rollback())
}
func TestStatisticsRollbackDoesNotPublish(t *testing.T) {
for _, kind := range []string{"peer", "interface"} {
for _, failure := range []string{"save", "commit"} {
t.Run(kind+"/"+failure, func(t *testing.T) {
collector, db, metrics := newStatisticsTestCollector(t)
if failure == "commit" {
pool, err := db.DB()
require.NoError(t, err)
db.Statement.ConnPool = &statisticsCommitFailurePool{DB: pool}
}
failed := make(chan struct{}, 1)
require.NoError(t, db.Callback().Update().Before("gorm:update").Register("test:reject-status", func(tx *gorm.DB) {
if failure == "save" {
tx.AddError(errors.New("injected status write failure"))
}
failed <- struct{}{}
}))
published := make(chan string, 4)
collector.bus = statisticsTestBus{publish: func(topic string, _ ...any) { published <- topic }}
startStatisticsTestCollector(t, collector, kind)
receiveStatisticsTest(t, failed)
// This read also waits for rollback to release the only connection.
var peer domain.PeerStatus
var iface domain.InterfaceStatus
require.NoError(t, db.First(&peer, "identifier = ?", "peer").Error)
require.NoError(t, db.First(&iface, "identifier = ?", "wg0").Error)
require.EqualValues(t, 2, peer.BytesReceived)
require.False(t, peer.IsConnected)
require.EqualValues(t, 1, iface.BytesTransmitted)
select {
case topic := <-published:
t.Errorf("rolled back update published %s", topic)
case status := <-metrics.statuses:
t.Errorf("rolled back update scheduled metrics: %+v", status)
case <-time.After(100 * time.Millisecond):
}
})
}
}
}
@@ -0,0 +1,161 @@
package wireguard
import (
"context"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/go-pkgz/routegroup"
"github.com/gorilla/websocket"
"github.com/h44z/wg-portal/internal/adapters"
"github.com/h44z/wg-portal/internal/app"
"github.com/h44z/wg-portal/internal/app/api/v0/handlers"
"github.com/h44z/wg-portal/internal/domain"
"github.com/stretchr/testify/require"
evbus "github.com/vardius/message-bus"
)
// Only authentication is replaced; the real endpoint performs the ownership
// check, SQLite query, caching, and WebSocket writes for a non-admin user.
type statisticsWebsocketAuth struct{ handlers.Authenticator }
func (statisticsWebsocketAuth) LoggedIn(...handlers.Scope) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
next.ServeHTTP(w, r.WithContext(domain.SetUserInfo(r.Context(), &domain.ContextUserInfo{Id: "user"})))
})
}
}
func (statisticsWebsocketAuth) RequireNonDefaultPassword() func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return next
}
}
type statisticsWebsocketBus struct {
evbus.MessageBus
ready chan struct{}
}
func (b statisticsWebsocketBus) Subscribe(topic string, fn any) error {
err := b.MessageBus.Subscribe(topic, fn)
// The endpoint subscribes to peer stats before interface stats.
if err == nil && topic == app.TopicInterfaceStatsUpdated {
close(b.ready)
}
return err
}
type statisticsOwnershipService struct {
*adapters.SqlRepo
entered chan struct{}
release chan struct{}
result chan error
once *sync.Once
}
func (s statisticsOwnershipService) GetPeer(ctx context.Context, id domain.PeerIdentifier) (*domain.Peer, error) {
s.once.Do(func() { close(s.entered) })
select {
case <-s.release:
case <-ctx.Done():
return nil, ctx.Err()
}
// Bound the old deadlock as well, so a failing regression cleans up all workers.
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
peer, err := s.SqlRepo.GetPeer(ctx, id)
s.result <- err
return peer, err
}
type statisticsBackpressureRepo struct {
*statisticsTestRepo
bus evbus.MessageBus
ownership statisticsOwnershipService
}
func (r statisticsBackpressureRepo) UpdatePeerStatus(ctx context.Context, id domain.PeerIdentifier,
update func(*domain.PeerStatus) (*domain.PeerStatus, error)) error {
return r.SqlRepo.UpdatePeerStatus(ctx, id, func(status *domain.PeerStatus) (*domain.PeerStatus, error) {
// The transaction now owns SQLite's only connection. Start the real
// ownership handler, park it, then fill its one-slot subscriber queue.
event := domain.TrafficDelta{EntityId: string(id)}
r.bus.Publish(app.TopicPeerStatsUpdated, event)
select {
case <-r.ownership.entered:
case <-ctx.Done():
return nil, ctx.Err()
}
r.bus.Publish(app.TopicPeerStatsUpdated, event)
close(r.ownership.release)
// The collector's third Publish must happen after this transaction returns.
return update(status)
})
}
func TestStatisticsBoundedBusWithNonAdminWebsocket(t *testing.T) {
collector, db, metrics := newStatisticsTestCollector(t)
repo := collector.db.(*statisticsTestRepo)
bus := statisticsWebsocketBus{MessageBus: evbus.New(1), ready: make(chan struct{})}
ownership := statisticsOwnershipService{SqlRepo: repo.SqlRepo,
entered: make(chan struct{}), release: make(chan struct{}), result: make(chan error, 4), once: &sync.Once{}}
collector.db = statisticsBackpressureRepo{statisticsTestRepo: repo, bus: bus, ownership: ownership}
collector.bus = bus
state := make(chan domain.PeerStatus, 1)
require.NoError(t, bus.Subscribe(app.TopicPeerStateChanged, func(status domain.PeerStatus, _ domain.Peer) { state <- status }))
defer bus.Close(app.TopicPeerStateChanged)
mux := http.NewServeMux()
endpointDone := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mux.ServeHTTP(w, r)
close(endpointDone)
}))
t.Cleanup(server.Close)
collector.cfg.Web.ExternalUrl = server.URL
endpoint := handlers.NewWebsocketEndpoint(collector.cfg, statisticsWebsocketAuth{}, bus, ownership)
endpoint.RegisterRoutes(routegroup.New(mux))
conn, _, err := websocket.DefaultDialer.Dial("ws"+server.URL[len("http"):]+"/ws",
http.Header{"Origin": []string{server.URL}})
require.NoError(t, err)
t.Cleanup(func() { conn.Close(); receiveStatisticsTest(t, endpointDone) })
receiveStatisticsTest(t, bus.ready)
// A separate reader models login/API traffic competing for the same pool.
readResult := make(chan error, 1)
var readers sync.WaitGroup
readers.Add(1)
go func() {
defer readers.Done()
<-ownership.entered
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
var user domain.User
readResult <- db.WithContext(ctx).First(&user, "identifier = ?", "user").Error
}()
t.Cleanup(readers.Wait)
startStatisticsTestCollector(t, collector, "peer")
require.NoError(t, receiveStatisticsTest(t, readResult), "concurrent API query must not be starved by a blocked Publish")
require.NoError(t, receiveStatisticsTest(t, ownership.result), "ownership query must not wait on the publishing transaction")
require.NoError(t, conn.SetReadDeadline(time.Now().Add(time.Second)))
for range 3 {
var message struct {
Type string `json:"type"`
Data domain.TrafficDelta `json:"data"`
}
require.NoError(t, conn.ReadJSON(&message))
require.Equal(t, "peer_stats", message.Type)
require.Equal(t, "peer", message.Data.EntityId)
}
require.True(t, receiveStatisticsTest(t, state).IsConnected)
receiveStatisticsTest(t, metrics.statuses)
var status domain.PeerStatus
require.NoError(t, db.First(&status, "identifier = ?", "peer").Error)
require.EqualValues(t, 42, status.BytesReceived)
}