database: Store xpub keys in a bucket.

**Warning: This commit contains a database upgrade.**

In order to add future support for retiring xpub keys, the database is
upgraded such that the keys are now stored in a dedicated bucket which
can hold multiple values rather than storing a single key as individual
values in the root bucket.

A new ID field is added to distinguish between keys. This ID is added to
every ticket record in the database in order to track which pubkey was
used for each ticket.

A new field named "Retired" has also been added to pubkeys. It is a unix
timestamp representing the moment the key was retired, or zero for the
currently active key.

database/database.go

@@ -39,14 +39,13 @@ var (
 	voteChangeBktK = []byte("votechangebkt")
 	// version is the current database version.
 	versionK = []byte("version")
-	// feeXPub is the extended public key used for collecting VSP fees.
-	feeXPubK = []byte("feeXPub")
+	// xPubBktK stores current and historic extended public keys used for
+	// collecting VSP fees.
+	xPubBktK = []byte("xpubbkt")
 	// cookieSecret is the secret key for initializing the cookie store.
 	cookieSecretK = []byte("cookieSecret")
 	// privatekey is the private key.
 	privateKeyK = []byte("privatekey")
-	// lastaddressindex is the index of the last address used for fees.
-	lastAddressIndexK = []byte("lastaddressindex")
 	// altSignAddrBktK stores alternate signing addresses.
 	altSignAddrBktK = []byte("altsigbkt")
 )
@@ -137,12 +136,14 @@ func CreateNew(dbFile, feeXPub string) error {
 			return err
 		}
 
-		// Store fee xpub.
-		xpub := FeeXPub{
+		// Insert the initial fee xpub with ID 0.
+		newKey := FeeXPub{
+			ID:          0,
 			Key:         feeXPub,
 			LastUsedIdx: 0,
+			Retired:     0,
 		}
-		err = insertFeeXPub(tx, xpub)
+		err = insertFeeXPub(tx, newKey)
 		if err != nil {
 			return err
 		}

database/feexpub.go

@@ -5,14 +5,20 @@
 package database
 
 import (
+	"encoding/json"
 	"fmt"
 
 	bolt "go.etcd.io/bbolt"
 )
 
+// FeeXPub is serialized to json and stored in bbolt db.
 type FeeXPub struct {
-	Key         string
-	LastUsedIdx uint32
+	ID          uint32 `json:"id"`
+	Key         string `json:"key"`
+	LastUsedIdx uint32 `json:"lastusedidx"`
+	// Retired is a unix timestamp representing the moment the key was retired,
+	// or zero for the currently active key.
+	Retired int64 `json:"retired"`
 }
 
 // insertFeeXPub stores the provided pubkey in the database, regardless of
@@ -20,43 +26,74 @@ type FeeXPub struct {
 func insertFeeXPub(tx *bolt.Tx, xpub FeeXPub) error {
 	vspBkt := tx.Bucket(vspBktK)
 
-	err := vspBkt.Put(feeXPubK, []byte(xpub.Key))
+	keyBkt, err := vspBkt.CreateBucketIfNotExists(xPubBktK)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get %s bucket: %w", string(xPubBktK), err)
 	}
 
-	return vspBkt.Put(lastAddressIndexK, uint32ToBytes(xpub.LastUsedIdx))
+	keyBytes, err := json.Marshal(xpub)
+	if err != nil {
+		return fmt.Errorf("could not marshal xpub: %w", err)
+	}
+
+	err = keyBkt.Put(uint32ToBytes(xpub.ID), keyBytes)
+	if err != nil {
+		return fmt.Errorf("could not store xpub: %w", err)
+	}
+
+	return nil
 }
 
-// FeeXPub retrieves the extended pubkey used for generating fee addresses
-// from the database.
+// FeeXPub retrieves the currently active extended pubkey used for generating
+// fee addresses from the database.
 func (vdb *VspDatabase) FeeXPub() (FeeXPub, error) {
-	var feeXPub string
-	var idx uint32
+	xpubs, err := vdb.AllXPubs()
+	if err != nil {
+		return FeeXPub{}, err
+	}
+
+	// Find the active xpub - the one with the highest ID.
+	var highest uint32
+	for id := range xpubs {
+		if id > highest {
+			highest = id
+		}
+	}
+
+	return xpubs[highest], nil
+}
+
+// AllXPubs retrieves the current and any retired extended pubkeys from the
+// database.
+func (vdb *VspDatabase) AllXPubs() (map[uint32]FeeXPub, error) {
+	xpubs := make(map[uint32]FeeXPub)
+
 	err := vdb.db.View(func(tx *bolt.Tx) error {
-		vspBkt := tx.Bucket(vspBktK)
+		bkt := tx.Bucket(vspBktK).Bucket(xPubBktK)
 
-		// Get the key.
-		xpubBytes := vspBkt.Get(feeXPubK)
-		if xpubBytes == nil {
-			return nil
+		if bkt == nil {
+			return fmt.Errorf("%s bucket doesn't exist", string(xPubBktK))
 		}
-		feeXPub = string(xpubBytes)
 
-		// Get the last used address index.
-		idxBytes := vspBkt.Get(lastAddressIndexK)
-		if idxBytes == nil {
-			return nil
+		err := bkt.ForEach(func(k, v []byte) error {
+			var xpub FeeXPub
+			err := json.Unmarshal(v, &xpub)
+			if err != nil {
+				return fmt.Errorf("could not unmarshal xpub key: %w", err)
 			}
-		idx = bytesToUint32(idxBytes)
+
+			xpubs[bytesToUint32(k)] = xpub
 
 			return nil
 		})
 		if err != nil {
-		return FeeXPub{}, fmt.Errorf("could not retrieve fee xpub: %w", err)
+			return fmt.Errorf("error iterating over %s bucket: %w", string(xPubBktK), err)
 		}
 
-	return FeeXPub{Key: feeXPub, LastUsedIdx: idx}, nil
+		return nil
+	})
+
+	return xpubs, err
 }
 
 // SetLastAddressIndex updates the last index used to derive a new fee address

database/feexpub_test.go

@@ -9,8 +9,7 @@ import (
 )
 
 func testFeeXPub(t *testing.T) {
-	// A newly created DB should store the fee xpub it was initialized with, and
-	// the last used index should be 0.
+	// A newly created DB should store the fee xpub it was initialized with.
 	retrievedXPub, err := db.FeeXPub()
 	if err != nil {
 		t.Fatalf("error getting fee xpub: %v", err)
@@ -20,8 +19,15 @@ func testFeeXPub(t *testing.T) {
 		t.Fatalf("expected fee xpub %v, got %v", feeXPub, retrievedXPub.Key)
 	}
 
+	// The ID, last used index and retirement timestamp should all be 0
+	if retrievedXPub.ID != 0 {
+		t.Fatalf("expected xpub ID 0, got %d", retrievedXPub.ID)
+	}
 	if retrievedXPub.LastUsedIdx != 0 {
-		t.Fatalf("retrieved addr index value didnt match expected")
+		t.Fatalf("expected xpub last used 0, got %d", retrievedXPub.LastUsedIdx)
+	}
+	if retrievedXPub.Retired != 0 {
+		t.Fatalf("expected xpub retirement 0, got %d", retrievedXPub.Retired)
 	}
 
 	// Update address index.
@@ -34,9 +40,20 @@ func testFeeXPub(t *testing.T) {
 	// Check for updated value.
 	retrievedXPub, err = db.FeeXPub()
 	if err != nil {
-		t.Fatalf("error getting address index: %v", err)
+		t.Fatalf("error getting fee xpub: %v", err)
 	}
-	if idx != retrievedXPub.LastUsedIdx {
-		t.Fatalf("retrieved addr index value didnt match expected")
+	if retrievedXPub.LastUsedIdx != idx {
+		t.Fatalf("expected xpub last used %d, got %d", idx, retrievedXPub.LastUsedIdx)
+	}
+
+	// Key, ID and retirement timestamp should be unchanged.
+	if retrievedXPub.Key != feeXPub {
+		t.Fatalf("expected fee xpub %v, got %v", feeXPub, retrievedXPub.Key)
+	}
+	if retrievedXPub.ID != 0 {
+		t.Fatalf("expected xpub ID 0, got %d", retrievedXPub.ID)
+	}
+	if retrievedXPub.Retired != 0 {
+		t.Fatalf("expected xpub retirement 0, got %d", retrievedXPub.Retired)
 	}
 }

database/ticket.go

@@ -50,6 +50,7 @@ var (
 	hashK              = []byte("Hash")
 	purchaseHeightK    = []byte("PurchaseHeight")
 	commitmentAddressK = []byte("CommitmentAddress")
+	feeAddressXPubIDK  = []byte("FeeAddressXPubID")
 	feeAddressIndexK   = []byte("FeeAddressIndex")
 	feeAddressK        = []byte("FeeAddress")
 	feeAmountK         = []byte("FeeAmount")
@@ -69,6 +70,7 @@ type Ticket struct {
 	Hash              string
 	PurchaseHeight    int64
 	CommitmentAddress string
+	FeeAddressXPubID  uint32
 	FeeAddressIndex   uint32
 	FeeAddress        string
 	FeeAmount         int64
@@ -184,6 +186,9 @@ func putTicketInBucket(bkt *bolt.Bucket, ticket Ticket) error {
 	if err = bkt.Put(feeAddressIndexK, uint32ToBytes(ticket.FeeAddressIndex)); err != nil {
 		return err
 	}
+	if err = bkt.Put(feeAddressXPubIDK, uint32ToBytes(ticket.FeeAddressXPubID)); err != nil {
+		return err
+	}
 	if err = bkt.Put(feeAmountK, int64ToBytes(ticket.FeeAmount)); err != nil {
 		return err
 	}
@@ -216,6 +221,7 @@ func getTicketFromBkt(bkt *bolt.Bucket) (Ticket, error) {
 	ticket.Outcome = TicketOutcome(bkt.Get(outcomeK))
 
 	ticket.PurchaseHeight = bytesToInt64(bkt.Get(purchaseHeightK))
+	ticket.FeeAddressXPubID = bytesToUint32(bkt.Get(feeAddressXPubIDK))
 	ticket.FeeAddressIndex = bytesToUint32(bkt.Get(feeAddressIndexK))
 	ticket.FeeAmount = bytesToInt64(bkt.Get(feeAmountK))
 	ticket.FeeExpiration = bytesToInt64(bkt.Get(feeExpirationK))

database/ticket_test.go

@@ -17,6 +17,7 @@ func exampleTicket() Ticket {
 		Hash:              randString(64, hexCharset),
 		CommitmentAddress: randString(35, addrCharset),
 		FeeAddressIndex:   12345,
+		FeeAddressXPubID:  10,
 		FeeAddress:        randString(35, addrCharset),
 		FeeAmount:         10000000,
 		FeeExpiration:     4,
@@ -129,6 +130,7 @@ func testUpdateTicket(t *testing.T) {
 	ticket.FeeAmount = ticket.FeeAmount + 1
 	ticket.FeeExpiration = ticket.FeeExpiration + 1
 	ticket.VoteChoices = map[string]string{"New agenda": "New value"}
+	ticket.FeeAddressXPubID = 20
 
 	err = db.UpdateTicket(ticket)
 	if err != nil {

database/upgrade_v5.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2024 The Decred developers
+// Use of this source code is governed by an ISC
+// license that can be found in the LICENSE file.
+
+package database
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/decred/slog"
+	bolt "go.etcd.io/bbolt"
+)
+
+func xPubBucketUpgrade(db *bolt.DB, log slog.Logger) error {
+	log.Infof("Upgrading database to version %d", xPubBucketVersion)
+
+	// feeXPub is the key which was used prior to this upgrade to store the xpub
+	// in the root bucket.
+	feeXPubK := []byte("feeXPub")
+
+	// lastaddressindex is the key which was used prior to this upgrade to store
+	// the index of the last used address in the root bucket.
+	lastAddressIndexK := []byte("lastaddressindex")
+
+	// Run the upgrade in a single database transaction so it can be safely
+	// rolled back if an error is encountered.
+	err := db.Update(func(tx *bolt.Tx) error {
+		vspBkt := tx.Bucket(vspBktK)
+		ticketBkt := vspBkt.Bucket(ticketBktK)
+
+		// Retrieve the current xpub.
+		xpubBytes := vspBkt.Get(feeXPubK)
+		if xpubBytes == nil {
+			return errors.New("xpub not found")
+		}
+		feeXPub := string(xpubBytes)
+
+		// Retrieve the current last addr index. Could be nil if this xpub was
+		// never used.
+		idxBytes := vspBkt.Get(lastAddressIndexK)
+		var idx uint32
+		if idxBytes != nil {
+			idx = bytesToUint32(idxBytes)
+		}
+
+		// Delete the old values from the database.
+		err := vspBkt.Delete(feeXPubK)
+		if err != nil {
+			return fmt.Errorf("could not delete xpub: %w", err)
+		}
+		err = vspBkt.Delete(lastAddressIndexK)
+		if err != nil {
+			return fmt.Errorf("could not delete last addr idx: %w", err)
+		}
+
+		// Insert the key into the bucket.
+		newXpub := FeeXPub{
+			ID:          0,
+			Key:         feeXPub,
+			LastUsedIdx: idx,
+			Retired:     0,
+		}
+		err = insertFeeXPub(tx, newXpub)
+		if err != nil {
+			return fmt.Errorf("failed to store xpub in new bucket: %w", err)
+		}
+
+		// Update all existing tickets with xpub key ID 0.
+		err = ticketBkt.ForEachBucket(func(k []byte) error {
+			return ticketBkt.Bucket(k).Put(feeAddressXPubIDK, uint32ToBytes(0))
+		})
+		if err != nil {
+			return fmt.Errorf("setting ticket xpub ID to 0 failed: %w", err)
+		}
+
+		// Update database version.
+		err = vspBkt.Put(versionK, uint32ToBytes(xPubBucketVersion))
+		if err != nil {
+			return fmt.Errorf("failed to update db version: %w", err)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	log.Info("Upgrade completed")
+
+	return nil
+}

database/upgrades.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021-2022 The Decred developers
+// Copyright (c) 2021-2024 The Decred developers
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
@@ -30,10 +30,17 @@ const (
 	// to verify messages sent to the vspd.
 	altSignAddrVersion = 4
 
+	// xPubBucketVersion changes how the xpub key and its associated addr index
+	// are stored. Previously only a single key was supported because it was
+	// stored as a single value in the root bucket. Now a dedicated bucket which
+	// can hold multiple keys is used, enabling support for historic retired
+	// keys as well as the current key.
+	xPubBucketVersion = 5
+
 	// latestVersion is the latest version of the database that is understood by
 	// vspd. Databases with recorded versions higher than this will fail to open
 	// (meaning any upgrades prevent reverting to older software).
-	latestVersion = altSignAddrVersion
+	latestVersion = xPubBucketVersion
 )
 
 // upgrades maps between old database versions and the upgrade function to
@@ -42,6 +49,7 @@ var upgrades = []func(tx *bolt.DB, log slog.Logger) error{
 	initialVersion:        removeOldFeeTxUpgrade,
 	removeOldFeeTxVersion: ticketBucketUpgrade,
 	ticketBucketVersion:   altSignAddrUpgrade,
+	altSignAddrVersion:    xPubBucketUpgrade,
 }
 
 // v1Ticket has the json tags required to unmarshal tickets stored in the

internal/webapi/addressgenerator.go

@@ -18,6 +18,7 @@ type addressGenerator struct {
 	external      *hdkeychain.ExtendedKey
 	netParams     *chaincfg.Params
 	lastUsedIndex uint32
+	feeXPubID     uint32
 	log           slog.Logger
 }
 
@@ -41,10 +42,15 @@ func newAddressGenerator(xPub database.FeeXPub, netParams *chaincfg.Params, log
 		external:      external,
 		netParams:     netParams,
 		lastUsedIndex: xPub.LastUsedIdx,
+		feeXPubID:     xPub.ID,
 		log:           log,
 	}, nil
 }
 
+func (m *addressGenerator) xPubID() uint32 {
+	return m.feeXPubID
+}
+
 // nextAddress increments the last used address counter and returns a new
 // address. It will skip any address index which causes an ErrInvalidChild.
 // Not safe for concurrent access.

internal/webapi/getfeeaddress.go

@@ -195,6 +195,7 @@ func (w *WebAPI) feeAddress(c *gin.Context) {
 		PurchaseHeight:    purchaseHeight,
 		CommitmentAddress: commitmentAddress,
 		FeeAddressIndex:   newAddressIdx,
+		FeeAddressXPubID:  w.addrGen.xPubID(),
 		FeeAddress:        newAddress,
 		Confirmed:         confirmed,
 		FeeAmount:         int64(fee),