Remove confirmed fee tx hex.

This PR introduces a database migration which deletes any raw fee transactions which remain in the database after already having been confirmed on-chain. There is no need to keep these, and they take up a lot of space.

There is also a change in the background handler to ensure that in future, tx hex is always removed from the database when the tx is confirmed.

background/background.go

@@ -180,6 +180,8 @@ func blockConnected() {
 		// If fee is confirmed, update the database and add ticket to voting
 		// wallets.
 		if feeTx.Confirmations >= requiredConfs {
+			// We no longer need the hex once the tx is confirmed on-chain.
+			ticket.FeeTxHex = ""
 			ticket.FeeTxStatus = database.FeeConfirmed
 			err = db.UpdateTicket(ticket)
 			if err != nil {

database/database.go

@@ -203,11 +203,11 @@ func Open(ctx context.Context, shutdownWg *sync.WaitGroup, dbFile string, backup
 		return nil, fmt.Errorf("unable to get db version: %w", err)
 	}
 
-	log.Debugf("Opened database (version=%d, file=%s)", dbVersion, dbFile)
+	log.Infof("Opened database (version=%d, file=%s)", dbVersion, dbFile)
 
 	err = vdb.Upgrade(dbVersion)
 	if err != nil {
-		return nil, fmt.Errorf("database upgrade failed: %w", err)
+		return nil, fmt.Errorf("upgrade failed: %w", err)
 	}
 
 	// Start a ticker to update the backup file at the specified interval.

database/database_upgrades.go

@@ -2,6 +2,8 @@ package database
 
 import (
 	"fmt"
+
+	bolt "go.etcd.io/bbolt"
 )
 
 const (
@@ -9,13 +11,23 @@ const (
 	// no upgrades applied.
 	initialVersion = 1
 
-	// latestVersion is the latest version of the bolt database that is
-	// understood by vspd. Databases with recorded versions higher than
-	// this will fail to open (meaning any upgrades prevent reverting to older
-	// software).
-	latestVersion = initialVersion
+	// removeOldFeeTxVersion deletes any raw fee transactions which remain in
+	// the database after already having been confirmed on-chain. There is no
+	// need to keep these, and they take up a lot of space.
+	removeOldFeeTxVersion = 2
+
+	// latestVersion is the latest version of the database that is understood by
+	// vspd. Databases with recorded versions higher than this will fail to open
+	// (meaning any upgrades prevent reverting to older software).
+	latestVersion = removeOldFeeTxVersion
 )
 
+// upgrades maps between old database versions and the upgrade function to
+// upgrade the database to the next version.
+var upgrades = []func(tx *bolt.DB) error{
+	initialVersion: removeOldFeeTxUpgrade,
+}
+
 // Upgrade will update the database to the latest known version.
 func (vdb *VspDatabase) Upgrade(currentVersion uint32) error {
 	if currentVersion == latestVersion {
@@ -25,7 +37,16 @@ func (vdb *VspDatabase) Upgrade(currentVersion uint32) error {
 
 	if currentVersion > latestVersion {
 		// Database is too new.
-		return fmt.Errorf("expected database version <= %d, got %d", latestVersion, currentVersion)
+		return fmt.Errorf("expected database version <= %d, got %d",
+			latestVersion, currentVersion)
+	}
+
+	// Execute all necessary upgrades in order.
+	for _, upgrade := range upgrades[currentVersion:] {
+		err := upgrade(vdb.db)
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil

database/v2_upgrade.go

@@ -0,0 +1,65 @@
+package database
+
+import (
+	"encoding/json"
+	"fmt"
+
+	bolt "go.etcd.io/bbolt"
+)
+
+func removeOldFeeTxUpgrade(db *bolt.DB) error {
+	log.Infof("Upgrading database to version %d", removeOldFeeTxVersion)
+
+	// Run the upgrade in a single database transaction so it can be safely
+	// rolled back if an error is encountered.
+	err := db.Update(func(tx *bolt.Tx) error {
+		vspBkt := tx.Bucket(vspBktK)
+		ticketBkt := vspBkt.Bucket(ticketBktK)
+
+		count := 0
+		err := ticketBkt.ForEach(func(k, v []byte) error {
+			// Deserialize the old ticket.
+			var ticket Ticket
+			err := json.Unmarshal(v, &ticket)
+			if err != nil {
+				return fmt.Errorf("could not unmarshal ticket: %w", err)
+			}
+
+			// Remove the fee tx hex if the tx is already confirmed.
+			if ticket.FeeTxStatus == FeeConfirmed {
+				count++
+				ticket.FeeTxHex = ""
+				ticketBytes, err := json.Marshal(ticket)
+				if err != nil {
+					return fmt.Errorf("could not marshal ticket: %w", err)
+				}
+
+				err = ticketBkt.Put(k, ticketBytes)
+				if err != nil {
+					return fmt.Errorf("could not put updated ticket: %w", err)
+				}
+			}
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		log.Infof("Dropped %d unnecessary raw transactions", count)
+
+		// Update database version.
+		err = vspBkt.Put(versionK, uint32ToBytes(removeOldFeeTxVersion))
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	log.Info("Upgrade completed")
+	return nil
+}