fix: pfsense backend (#703)

* Return empty string instead of "<nil>" when a genericjsonobject key doesn't exist.

* Fix pfsense backend

* Fix API request parameter names and types
* Refactor interface and peer creation to send the necessary parameters
* Automatically call apply when interfaces or peers are changed

Signed-off-by: Aram Akhavan <1147328+kaysond@users.noreply.github.com>

---------

Signed-off-by: Aram Akhavan <1147328+kaysond@users.noreply.github.com>

docs/documentation/usage/backends.md

@@ -61,7 +61,7 @@ backend:
 
 > :warning: The pfSense backend is currently **alpha**. Only basic interface and peer CRUD are supported. Traffic statistics (rx/tx, last handshake) are not exposed by the pfSense REST API and will show as empty.
 
-The pfSense backend talks to the pfSense REST API (pfSense Plus / CE with the REST API package installed). Point the backend at the appliance hostname without appending `/api/v2` — the portal appends `/api/v2` automatically.
+The pfSense backend talks to the pfSense REST API (pfSense Plus / CE with the REST API package installed). Point the backend at the appliance hostname without appending `/api/v2` — the portal appends `/api/v2` automatically. wg-portal is developed for and tested against REST API v2.8.0.
 
 ### Prerequisites on pfSense:
 - pfSense with the REST API package enabled (`System -> API`) and WireGuard configured.

internal/adapters/wgcontroller/pfsense.go

@@ -617,18 +617,22 @@ func (c *PfsenseController) SaveInterface(
 	mutex.Lock()
 	defer mutex.Unlock()
 
-	physicalInterface, err := c.getOrCreateInterface(ctx, id)
+	physicalInterface, err := c.getInterface(ctx, id)
 	if err != nil {
 		return err
 	}
 
-	deviceId := ""
+	if physicalInterface == nil {
-	if physicalInterface.GetExtras() != nil {
+		physicalInterface = &domain.PhysicalInterface{
-		if extras, ok := physicalInterface.GetExtras().(domain.PfsenseInterfaceExtras); ok {
+			Identifier:   id,
-			deviceId = extras.Id
+			ImportSource: domain.ControllerTypePfsense,
+			DeviceType:   domain.ControllerTypePfsense,
 		}
+		physicalInterface.SetExtras(domain.PfsenseInterfaceExtras{})
 	}
 
+	deviceId := physicalInterface.GetExtras().(domain.PfsenseInterfaceExtras).Id
+
 	if updateFunc != nil {
 		physicalInterface, err = updateFunc(physicalInterface)
 		if err != nil {
@@ -643,14 +647,14 @@ func (c *PfsenseController) SaveInterface(
 		}
 	}
 
-	if err := c.updateInterface(ctx, physicalInterface); err != nil {
+	if err := c.createOrUpdateInterface(ctx, physicalInterface); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (c *PfsenseController) getOrCreateInterface(
+func (c *PfsenseController) getInterface(
 	ctx context.Context,
 	id domain.InterfaceIdentifier,
 ) (*domain.PhysicalInterface, error) {
@@ -659,50 +663,84 @@ func (c *PfsenseController) getOrCreateInterface(
 			"name": string(id),
 		},
 	})
-	if wgReply.Status == lowlevel.PfsenseApiStatusOk && len(wgReply.Data) > 0 {
+	if wgReply.Status != lowlevel.PfsenseApiStatusOk {
-		return c.loadInterfaceData(ctx, wgReply.Data[0])
+		return nil, fmt.Errorf("failed to query interface %s: %v", id, wgReply.Error)
 	}
-
+	if len(wgReply.Data) == 0 {
-	// create a new tunnel if it does not exist
+		return nil, nil
-	// Actual endpoint: POST /api/v2/vpn/wireguard/tunnel (singular)
-	createReply := c.client.Create(ctx, "/api/v2/vpn/wireguard/tunnel", lowlevel.GenericJsonObject{
-		"name": string(id),
-	})
-	if createReply.Status == lowlevel.PfsenseApiStatusOk {
-		return c.loadInterfaceData(ctx, createReply.Data)
 	}
-
+	return c.loadInterfaceData(ctx, wgReply.Data[0])
-	return nil, fmt.Errorf("failed to create interface %s: %v", id, createReply.Error)
 }
 
-func (c *PfsenseController) updateInterface(ctx context.Context, pi *domain.PhysicalInterface) error {
+type pfsenseWireGuardAddress struct {
+	Address string `json:"address"`
+	Mask    int    `json:"mask"`
+	Descr   string `json:"descr"`
+}
+
+func cidrToPfsense(cidr *domain.Cidr) pfsenseWireGuardAddress {
+	return pfsenseWireGuardAddress{
+		Address: cidr.Addr,
+		Mask:    cidr.NetLength,
+		// supported in pfsense, but not in wg-portal GUI
+		Descr: "",
+	}
+}
+
+func (c *PfsenseController) createOrUpdateInterface(ctx context.Context, pi *domain.PhysicalInterface) error {
 	extras := pi.GetExtras().(domain.PfsenseInterfaceExtras)
 	interfaceId := extras.Id
 
 	payload := lowlevel.GenericJsonObject{
-		"name":        string(pi.Identifier),
+		"name":       string(pi.Identifier),
-		"description": extras.Comment,
+		"descr":      extras.Comment,
-		"mtu":         strconv.Itoa(pi.Mtu),
+		"mtu":        pi.Mtu,
-		"listenport":  strconv.Itoa(pi.ListenPort),
+		"listenport": strconv.Itoa(pi.ListenPort),
-		"privatekey":  pi.KeyPair.PrivateKey,
+		"privatekey": pi.KeyPair.PrivateKey,
-		"disabled":    strconv.FormatBool(!pi.DeviceUp),
+		"disabled":   strconv.FormatBool(!pi.DeviceUp),
 	}
 
-	// Add addresses if present
+	addresses := make([]pfsenseWireGuardAddress, 0, len(pi.Addresses))
-	if len(pi.Addresses) > 0 {
+	for _, addr := range pi.Addresses {
-		addresses := make([]string, 0, len(pi.Addresses))
+		addresses = append(addresses, cidrToPfsense(&addr))
-		for _, addr := range pi.Addresses {
+	}
-			addresses = append(addresses, addr.String())
+	payload["addresses"] = addresses
+
+	if interfaceId == "" {
+		// Actual endpoint: POST /api/v2/vpn/wireguard/tunnel (singular)
+		createReply := c.client.Create(ctx, "/api/v2/vpn/wireguard/tunnel", payload)
+		if createReply.Status != lowlevel.PfsenseApiStatusOk {
+			return fmt.Errorf("failed to create interface %s: %v", pi.Identifier, createReply.Error)
 		}
-		payload["addresses"] = strings.Join(addresses, ",")
+		// Capture the newly-assigned ID so callers see it
+		if newId := createReply.Data.GetString("id"); newId != "" {
+			extras.Id = newId
+			pi.SetExtras(extras)
+		}
+		if applyReply := c.client.Create(ctx, "/api/v2/vpn/wireguard/apply", lowlevel.GenericJsonObject{}); applyReply.Status != lowlevel.PfsenseApiStatusOk {
+			return fmt.Errorf("failed to apply WireGuard changes after creating interface %s: %v",
+				pi.Identifier, applyReply.Error)
+		}
+		return nil
 	}
 
-	// Actual endpoint: PATCH /api/v2/vpn/wireguard/tunnel?id={id}
+	interfaceIdInt, err := strconv.Atoi(interfaceId)
-	wgReply := c.client.Update(ctx, "/api/v2/vpn/wireguard/tunnel?id="+interfaceId, payload)
+	if err != nil {
+		return fmt.Errorf("invalid pfSense interface id %q for %s: %w", interfaceId, pi.Identifier, err)
+	}
+	payload["id"] = interfaceIdInt
+
+	// Actual endpoint: PATCH /api/v2/vpn/wireguard/tunnel
+	wgReply := c.client.Update(ctx, "/api/v2/vpn/wireguard/tunnel", payload)
 	if wgReply.Status != lowlevel.PfsenseApiStatusOk {
 		return fmt.Errorf("failed to update interface %s: %v", pi.Identifier, wgReply.Error)
 	}
 
+	if applyReply := c.client.Create(ctx, "/api/v2/vpn/wireguard/apply", lowlevel.GenericJsonObject{}); applyReply.Status != lowlevel.PfsenseApiStatusOk {
+		return fmt.Errorf("failed to apply WireGuard changes after updating interface %s: %v",
+			pi.Identifier, applyReply.Error)
+	}
+
 	return nil
 }
 
@@ -726,8 +764,10 @@ func (c *PfsenseController) DeleteInterface(ctx context.Context, id domain.Inter
 	}
 
 	interfaceId := wgReply.Data[0].GetString("id")
-	// Actual endpoint: DELETE /api/v2/vpn/wireguard/tunnel?id={id}
+	// Actual endpoint: DELETE /api/v2/vpn/wireguard/tunnel?id={id}&apply=true
-	deleteReply := c.client.Delete(ctx, "/api/v2/vpn/wireguard/tunnel?id="+interfaceId)
+	deleteReply := c.client.Delete(ctx, "/api/v2/vpn/wireguard/tunnel", &lowlevel.PfsenseRequestOptions{
+		Filters: map[string]string{"id": interfaceId, "apply": "true"},
+	})
 	if deleteReply.Status != lowlevel.PfsenseApiStatusOk {
 		return fmt.Errorf("failed to delete WireGuard interface %s: %v", id, deleteReply.Error)
 	}
@@ -746,18 +786,22 @@ func (c *PfsenseController) SavePeer(
 	mutex.Lock()
 	defer mutex.Unlock()
 
-	physicalPeer, err := c.getOrCreatePeer(ctx, deviceId, id)
+	physicalPeer, err := c.getPeer(ctx, deviceId, id)
 	if err != nil {
 		return err
 	}
 
-	peerId := ""
+	if physicalPeer == nil {
-	if physicalPeer.GetExtras() != nil {
+		physicalPeer = &domain.PhysicalPeer{
-		if extras, ok := physicalPeer.GetExtras().(domain.PfsensePeerExtras); ok {
+			Identifier:   id,
-			peerId = extras.Id
+			KeyPair:      domain.KeyPair{PublicKey: string(id)},
+			ImportSource: domain.ControllerTypePfsense,
 		}
+		physicalPeer.SetExtras(domain.PfsensePeerExtras{})
 	}
 
+	peerId := physicalPeer.GetExtras().(domain.PfsensePeerExtras).Id
+
 	physicalPeer, err = updateFunc(physicalPeer)
 	if err != nil {
 		return err
@@ -770,14 +814,14 @@ func (c *PfsenseController) SavePeer(
 		}
 	}
 
-	if err := c.updatePeer(ctx, deviceId, physicalPeer); err != nil {
+	if err := c.createOrUpdatePeer(ctx, deviceId, physicalPeer); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (c *PfsenseController) getOrCreatePeer(
+func (c *PfsenseController) getPeer(
 	ctx context.Context,
 	deviceId domain.InterfaceIdentifier,
 	id domain.PeerIdentifier,
@@ -787,40 +831,25 @@ func (c *PfsenseController) getOrCreatePeer(
 	wgReply := c.client.Query(ctx, "/api/v2/vpn/wireguard/peers", &lowlevel.PfsenseRequestOptions{
 		Filters: map[string]string{
 			"publickey": string(id),
-			"tun":        string(deviceId), // Use "tun" field name as that's what the API uses
+			"tun":       string(deviceId), // Use "tun" field name as that's what the API uses
 		},
 	})
-	if wgReply.Status == lowlevel.PfsenseApiStatusOk && len(wgReply.Data) > 0 {
+	if wgReply.Status != lowlevel.PfsenseApiStatusOk {
-		slog.Debug("found existing pfSense peer", "peer", id, "interface", deviceId)
+		return nil, fmt.Errorf("failed to query peer %s for interface %s: %v", id, deviceId, wgReply.Error)
-		existingPeer, err := c.convertWireGuardPeer(wgReply.Data[0])
+	}
-		if err != nil {
+	if len(wgReply.Data) == 0 {
-			return nil, err
+		return nil, nil
-		}
-		return &existingPeer, nil
 	}
 
-	// create a new peer if it does not exist
+	slog.Debug("found existing pfSense peer", "peer", id, "interface", deviceId)
-	// Actual endpoint: POST /api/v2/vpn/wireguard/peer (singular)
+	existingPeer, err := c.convertWireGuardPeer(wgReply.Data[0])
-	slog.Debug("creating new pfSense peer", "peer", id, "interface", deviceId)
+	if err != nil {
-	createReply := c.client.Create(ctx, "/api/v2/vpn/wireguard/peer", lowlevel.GenericJsonObject{
+		return nil, err
-		"name":       fmt.Sprintf("wg-%s", id[0:8]),
-		"interface": string(deviceId),
-		"publickey": string(id),
-		"allowedips": "0.0.0.0/0", // Use 0.0.0.0/0 as default, will be updated by updatePeer
-	})
-	if createReply.Status == lowlevel.PfsenseApiStatusOk {
-		newPeer, err := c.convertWireGuardPeer(createReply.Data)
-		if err != nil {
-			return nil, err
-		}
-		slog.Debug("successfully created pfSense peer", "peer", id, "interface", deviceId)
-		return &newPeer, nil
 	}
-
+	return &existingPeer, nil
-	return nil, fmt.Errorf("failed to create peer %s for interface %s: %v", id, deviceId, createReply.Error)
 }
 
-func (c *PfsenseController) updatePeer(
+func (c *PfsenseController) createOrUpdatePeer(
 	ctx context.Context,
 	deviceId domain.InterfaceIdentifier,
 	pp *domain.PhysicalPeer,
@@ -828,36 +857,74 @@ func (c *PfsenseController) updatePeer(
 	extras := pp.GetExtras().(domain.PfsensePeerExtras)
 	peerId := extras.Id
 
-	allowedIPsStr := domain.CidrsToString(pp.AllowedIPs)
-
-	slog.Debug("updating pfSense peer",
-		"peer", pp.Identifier,
-		"interface", deviceId,
-		"allowed-ips", allowedIPsStr,
-		"allowed-ips-count", len(pp.AllowedIPs),
-		"disabled", extras.Disabled)
-
 	payload := lowlevel.GenericJsonObject{
-		"name":                 extras.Name,
+		"tun":                 string(deviceId),
-		"description":          extras.Comment,
+		"descr":               extras.Name,
-		"presharedkey":         string(pp.PresharedKey),
+		"presharedkey":        string(pp.PresharedKey),
-		"publickey":            pp.KeyPair.PublicKey,
+		"publickey":           pp.KeyPair.PublicKey,
-		"privatekey":           pp.KeyPair.PrivateKey,
+		"persistentkeepalive": pp.PersistentKeepalive,
-		"persistentkeepalive":  strconv.Itoa(pp.PersistentKeepalive),
+		"enabled":             !extras.Disabled,
-		"disabled":             strconv.FormatBool(extras.Disabled),
-		"allowedips":           allowedIPsStr,
 	}
 
 	if pp.Endpoint != "" {
 		payload["endpoint"] = pp.Endpoint
 	}
 
-	// Actual endpoint: PATCH /api/v2/vpn/wireguard/peer?id={id}
+	allowedIps := make([]pfsenseWireGuardAddress, 0, len(pp.AllowedIPs))
-	wgReply := c.client.Update(ctx, "/api/v2/vpn/wireguard/peer?id="+peerId, payload)
+	for _, addr := range pp.AllowedIPs {
+		allowedIps = append(allowedIps, cidrToPfsense(&addr))
+	}
+	payload["allowedips"] = allowedIps
+
+	if peerId == "" {
+		slog.Debug("creating new pfSense peer",
+			"peer", pp.Identifier,
+			"interface", deviceId,
+			"allowed-ips", domain.CidrsToString(pp.AllowedIPs),
+			"disabled", extras.Disabled)
+
+		// Actual endpoint: POST /api/v2/vpn/wireguard/peer (singular)
+		createReply := c.client.Create(ctx, "/api/v2/vpn/wireguard/peer", payload)
+		if createReply.Status != lowlevel.PfsenseApiStatusOk {
+			return fmt.Errorf("failed to create peer %s for interface %s: %v",
+				pp.Identifier, deviceId, createReply.Error)
+		}
+		if newId := createReply.Data.GetString("id"); newId != "" {
+			extras.Id = newId
+			pp.SetExtras(extras)
+		}
+		if applyReply := c.client.Create(ctx, "/api/v2/vpn/wireguard/apply", lowlevel.GenericJsonObject{}); applyReply.Status != lowlevel.PfsenseApiStatusOk {
+			return fmt.Errorf("failed to apply WireGuard changes after creating peer %s on interface %s: %v",
+				pp.Identifier, deviceId, applyReply.Error)
+		}
+		slog.Debug("successfully created pfSense peer", "peer", pp.Identifier, "interface", deviceId)
+		return nil
+	}
+
+	slog.Debug("updating pfSense peer",
+		"peer", pp.Identifier,
+		"interface", deviceId,
+		"allowed-ips", domain.CidrsToString(pp.AllowedIPs),
+		"allowed-ips-count", len(pp.AllowedIPs),
+		"disabled", extras.Disabled)
+
+	peerIdInt, err := strconv.Atoi(peerId)
+	if err != nil {
+		return fmt.Errorf("invalid pfSense peer id %q for %s: %w", peerId, pp.Identifier, err)
+	}
+	payload["id"] = peerIdInt
+
+	// Actual endpoint: PATCH /api/v2/vpn/wireguard/peer
+	wgReply := c.client.Update(ctx, "/api/v2/vpn/wireguard/peer", payload)
 	if wgReply.Status != lowlevel.PfsenseApiStatusOk {
 		return fmt.Errorf("failed to update peer %s on interface %s: %v", pp.Identifier, deviceId, wgReply.Error)
 	}
 
+	if applyReply := c.client.Create(ctx, "/api/v2/vpn/wireguard/apply", lowlevel.GenericJsonObject{}); applyReply.Status != lowlevel.PfsenseApiStatusOk {
+		return fmt.Errorf("failed to apply WireGuard changes after updating peer %s on interface %s: %v",
+			pp.Identifier, deviceId, applyReply.Error)
+	}
+
 	if extras.Disabled {
 		slog.Debug("successfully disabled pfSense peer", "peer", pp.Identifier, "interface", deviceId)
 	} else {
@@ -882,7 +949,7 @@ func (c *PfsenseController) DeletePeer(
 	wgReply := c.client.Query(ctx, "/api/v2/vpn/wireguard/peers", &lowlevel.PfsenseRequestOptions{
 		Filters: map[string]string{
 			"publickey": string(id),
-			"tun":        string(deviceId), // Use "tun" field name as that's what the API uses
+			"tun":       string(deviceId), // Use "tun" field name as that's what the API uses
 		},
 	})
 	if wgReply.Status != lowlevel.PfsenseApiStatusOk {
@@ -893,8 +960,10 @@ func (c *PfsenseController) DeletePeer(
 	}
 
 	peerId := wgReply.Data[0].GetString("id")
-	// Actual endpoint: DELETE /api/v2/vpn/wireguard/peer?id={id}
+	// Actual endpoint: DELETE /api/v2/vpn/wireguard/peer?id={id}&apply=true
-	deleteReply := c.client.Delete(ctx, "/api/v2/vpn/wireguard/peer?id="+peerId)
+	deleteReply := c.client.Delete(ctx, "/api/v2/vpn/wireguard/peer", &lowlevel.PfsenseRequestOptions{
+		Filters: map[string]string{"id": peerId, "apply": "true"},
+	})
 	if deleteReply.Status != lowlevel.PfsenseApiStatusOk {
 		return fmt.Errorf("failed to delete WireGuard peer %s for interface %s: %v", id, deviceId, deleteReply.Error)
 	}
@@ -976,4 +1045,3 @@ func (c *PfsenseController) PingAddresses(
 }
 
 // endregion statistics-related
-

internal/lowlevel/mikrotik.go

@@ -57,6 +57,9 @@ type EmptyResponse struct{}
 
 func (JsonObject GenericJsonObject) GetString(key string) string {
 	if value, ok := JsonObject[key]; ok {
+		if value == nil {
+			return ""
+		}
 		if strValue, ok := value.(string); ok {
 			return strValue
 		} else {

internal/lowlevel/pfsense.go

@@ -23,7 +23,7 @@ import (
 // region models
 
 const (
-	PfsenseApiStatusOk    = "ok"    // pfSense REST API uses "ok" in response
+	PfsenseApiStatusOk    = "ok" // pfSense REST API uses "ok" in response
 	PfsenseApiStatusError = "error"
 )
 
@@ -37,8 +37,8 @@ const (
 type PfsenseApiResponse[T any] struct {
 	Status string
 	Code   int
-	Data   T                 `json:"data,omitempty"`
+	Data   T                `json:"data,omitempty"`
-	Error  *PfsenseApiError  `json:"error,omitempty"`
+	Error  *PfsenseApiError `json:"error,omitempty"`
 }
 
 type PfsenseApiError struct {
@@ -193,6 +193,7 @@ func (p *PfsenseApiClient) preparePayloadRequest(
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal payload: %w", err)
 	}
+	p.debugLog("Prepared payload", "payload", string(payloadBytes))
 
 	req, err := http.NewRequestWithContext(ctx, method, fullUrl, bytes.NewReader(payloadBytes))
 	if err != nil {
@@ -405,11 +406,12 @@ func (p *PfsenseApiClient) Update(
 func (p *PfsenseApiClient) Delete(
 	ctx context.Context,
 	command string,
+	opts *PfsenseRequestOptions,
 ) PfsenseApiResponse[EmptyResponse] {
 	apiCtx, cancel := context.WithTimeout(ctx, p.cfg.GetApiTimeout())
 	defer cancel()
 
-	fullUrl := p.getFullPath(command)
+	fullUrl := opts.GetPath(p.getFullPath(command))
 
 	req, err := p.prepareDeleteRequest(apiCtx, fullUrl)
 	if err != nil {
@@ -425,4 +427,3 @@ func (p *PfsenseApiClient) Delete(
 }
 
 // endregion API-client
-