stakey/vspd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Periodically write a database backup. (#76)

Browse Source
This commit is contained in:
Jamie Holdstock 2020-05-28 06:58:34 +01:00 committed by GitHub
parent d275fddf1b
commit 6b6bc20522
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 99 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
vspd
/database/test.db
/database/test.db-backup

8
README.md
View File

@ -67,7 +67,13 @@ libraries:
## Backup
- Regular backups of bbolt database and feexpub.
The bbolt database file used by vspd is stored in the process home directory, at
the path `{homedir}/data/{network}/vspd.db`. vspd keeps a file lock on this
file, so it cannot be opened by any other processes while vspd is running.
To facilitate back-ups, vspd will write periodically write a copy of the bbolt
database to the path `{homedir}/data/{network}/vspd.db-backup`. A copy of the
database file will also be written to this path when vspd shuts down.
## Issue Tracker

43
config.go
View File

@ -10,6 +10,7 @@ import (
"path/filepath"
"runtime"
"strings"
"time"
"github.com/decred/dcrd/dcrutil/v3"
"github.com/decred/dcrd/hdkeychain/v3"
@ -27,27 +28,29 @@ var (
defaultDcrdHost = "127.0.0.1"
defaultWalletHost = "127.0.0.1"
defaultWebServerDebug = false
defaultBackupInterval = time.Minute * 3
)
// config defines the configuration options for the VSP.
type config struct {
Listen string `long:"listen" ini-name:"listen" description:"The ip:port to listen for API requests."`
LogLevel string `long:"loglevel" ini-name:"loglevel" description:"Logging level." choice:"trace" choice:"debug" choice:"info" choice:"warn" choice:"error" choice:"critical"`
Network string `long:"network" ini-name:"network" description:"Decred network to use." choice:"testnet" choice:"mainnet" choice:"simnet"`
FeeXPub string `long:"feexpub" ini-name:"feexpub" description:"Cold wallet xpub used for collecting fees."`
VSPFee float64 `long:"vspfee" ini-name:"vspfee" description:"Fee percentage charged for VSP use. eg. 0.01 (1%), 0.05 (5%)."`
HomeDir string `long:"homedir" ini-name:"homedir" no-ini:"true" description:"Path to application home directory. Used for storing VSP database and logs."`
ConfigFile string `long:"configfile" ini-name:"configfile" no-ini:"true" description:"Path to configuration file."`
DcrdHost string `long:"dcrdhost" ini-name:"dcrdhost" description:"The ip:port to establish a JSON-RPC connection with dcrd. Should be the same host where vspd is running."`
DcrdUser string `long:"dcrduser" ini-name:"dcrduser" description:"Username for dcrd RPC connections."`
DcrdPass string `long:"dcrdpass" ini-name:"dcrdpass" description:"Password for dcrd RPC connections."`
DcrdCert string `long:"dcrdcert" ini-name:"dcrdcert" description:"The dcrd RPC certificate file."`
WalletHost string `long:"wallethost" ini-name:"wallethost" description:"The ip:port to establish a JSON-RPC connection with voting dcrwallet."`
WalletUser string `long:"walletuser" ini-name:"walletuser" description:"Username for dcrwallet RPC connections."`
WalletPass string `long:"walletpass" ini-name:"walletpass" description:"Password for dcrwallet RPC connections."`
WalletCert string `long:"walletcert" ini-name:"walletcert" description:"The dcrwallet RPC certificate file."`
WebServerDebug bool `long:"webserverdebug" ini-name:"webserverdebug" description:"Enable web server debug mode (verbose logging to terminal and live-reloading templates)."`
SupportEmail string `long:"supportemail" ini-name:"supportemail" description:"Email address for users in need of support."`
Listen string `long:"listen" ini-name:"listen" description:"The ip:port to listen for API requests."`
LogLevel string `long:"loglevel" ini-name:"loglevel" description:"Logging level." choice:"trace" choice:"debug" choice:"info" choice:"warn" choice:"error" choice:"critical"`
Network string `long:"network" ini-name:"network" description:"Decred network to use." choice:"testnet" choice:"mainnet" choice:"simnet"`
FeeXPub string `long:"feexpub" ini-name:"feexpub" description:"Cold wallet xpub used for collecting fees."`
VSPFee float64 `long:"vspfee" ini-name:"vspfee" description:"Fee percentage charged for VSP use. eg. 0.01 (1%), 0.05 (5%)."`
HomeDir string `long:"homedir" ini-name:"homedir" no-ini:"true" description:"Path to application home directory. Used for storing VSP database and logs."`
ConfigFile string `long:"configfile" ini-name:"configfile" no-ini:"true" description:"Path to configuration file."`
DcrdHost string `long:"dcrdhost" ini-name:"dcrdhost" description:"The ip:port to establish a JSON-RPC connection with dcrd. Should be the same host where vspd is running."`
DcrdUser string `long:"dcrduser" ini-name:"dcrduser" description:"Username for dcrd RPC connections."`
DcrdPass string `long:"dcrdpass" ini-name:"dcrdpass" description:"Password for dcrd RPC connections."`
DcrdCert string `long:"dcrdcert" ini-name:"dcrdcert" description:"The dcrd RPC certificate file."`
WalletHost string `long:"wallethost" ini-name:"wallethost" description:"The ip:port to establish a JSON-RPC connection with voting dcrwallet."`
WalletUser string `long:"walletuser" ini-name:"walletuser" description:"Username for dcrwallet RPC connections."`
WalletPass string `long:"walletpass" ini-name:"walletpass" description:"Password for dcrwallet RPC connections."`
WalletCert string `long:"walletcert" ini-name:"walletcert" description:"The dcrwallet RPC certificate file."`
WebServerDebug bool `long:"webserverdebug" ini-name:"webserverdebug" description:"Enable web server debug mode (verbose logging to terminal and live-reloading templates)."`
SupportEmail string `long:"supportemail" ini-name:"supportemail" description:"Email address for users in need of support."`
BackupInterval time.Duration `long:"backupinterval" ini-name:"backupinterval" description:"Time period between automatic database backups. Valid time units are {s,m,h}. Minimum 30 seconds."`
dbPath string
netParams *netParams
@ -152,6 +155,7 @@ func loadConfig() (*config, error) {
DcrdHost: defaultDcrdHost,
WalletHost: defaultWalletHost,
WebServerDebug: defaultWebServerDebug,
BackupInterval: defaultBackupInterval,
}
// Pre-parse the command line options to see if an alternative config
@ -241,6 +245,11 @@ func loadConfig() (*config, error) {
cfg.netParams = &simNetParams
}
// Ensure backup interval is greater than 30 seconds.
if cfg.BackupInterval < time.Second*30 {
return nil, errors.New("minimum backupinterval is 30 seconds")
}
// Ensure the support email address is set.
if cfg.SupportEmail == "" {
return nil, errors.New("the supportemail option is not set")

59
database/database.go
View File

@ -6,6 +6,7 @@ import (
"crypto/rand"
"encoding/binary"
"fmt"
"os"
"sync"
"time"
@ -33,9 +34,38 @@ var (
lastAddressIndexK = []byte("lastaddressindex")
)
// backupMtx protects writeBackup, to ensure only one backup file is written at
// a time.
var backupMtx sync.Mutex
func writeBackup(db *bolt.DB, dbFile string) error {
backupMtx.Lock()
defer backupMtx.Unlock()
backupPath := dbFile + "-backup"
tempPath := backupPath + "~"
// Write backup to temporary file.
err := db.View(func(tx *bolt.Tx) error {
return tx.CopyFile(tempPath, 0600)
})
if err != nil {
return fmt.Errorf("tx.CopyFile: %v", err)
}
// Rename temporary file to actual backup file.
err = os.Rename(tempPath, backupPath)
if err != nil {
return fmt.Errorf("os.Rename: %v", err)
}
log.Debugf("Database backup written to %s", backupPath)
return err
}
// Open initializes and returns an open database. If no database file is found
// at the provided path, a new one will be created.
func Open(ctx context.Context, shutdownWg *sync.WaitGroup, dbFile string) (*VspDatabase, error) {
func Open(ctx context.Context, shutdownWg *sync.WaitGroup, dbFile string, backupInterval time.Duration) (*VspDatabase, error) {
db, err := bolt.Open(dbFile, 0600, &bolt.Options{Timeout: 1 * time.Second})
if err != nil {
@ -51,7 +81,13 @@ func Open(ctx context.Context, shutdownWg *sync.WaitGroup, dbFile string) (*VspD
<-ctx.Done()
log.Debug("Closing database...")
err := db.Close()
err := writeBackup(db, dbFile)
if err != nil {
log.Errorf("Failed to write database backup: %v", err)
}
err = db.Close()
if err != nil {
log.Errorf("Error closing database: %v", err)
} else {
@ -60,6 +96,25 @@ func Open(ctx context.Context, shutdownWg *sync.WaitGroup, dbFile string) (*VspD
shutdownWg.Done()
}()
// Start a ticker to update the backup file at the specified interval.
shutdownWg.Add(1)
backupTicker := time.NewTicker(backupInterval)
go func() {
for {
select {
case <-backupTicker.C:
err := writeBackup(db, dbFile)
if err != nil {
log.Errorf("Failed to write database backup: %v", err)
}
case <-ctx.Done():
backupTicker.Stop()
shutdownWg.Done()
return
}
}
}()
// Create all storage buckets of the VSP if they don't already exist.
err = db.Update(func(tx *bolt.Tx) error {
if tx.Bucket(vspBktK) == nil {

10
database/database_test.go
View File

@ -5,17 +5,20 @@ import (
"os"
"sync"
"testing"
"time"
)
var (
testDb = "test.db"
db *VspDatabase
testDb = "test.db"
backupDb = "test.db-backup"
db *VspDatabase
)
// TestDatabase runs all database tests.
func TestDatabase(t *testing.T) {
// Ensure we are starting with a clean environment.
os.Remove(testDb)
os.Remove(backupDb)
// All sub-tests to run.
tests := map[string]func(*testing.T){
@ -31,7 +34,7 @@ func TestDatabase(t *testing.T) {
var err error
var wg sync.WaitGroup
ctx, cancel := context.WithCancel(context.TODO())
db, err = Open(ctx, &wg, testDb)
db, err = Open(ctx, &wg, testDb, time.Hour)
if err != nil {
t.Fatalf("error creating test database: %v", err)
}
@ -44,6 +47,7 @@ func TestDatabase(t *testing.T) {
wg.Wait()
os.Remove(testDb)
os.Remove(backupDb)
}
}

2
main.go
View File

@ -49,7 +49,7 @@ func run(ctx context.Context) error {
defer log.Info("Shutdown complete")
// Open database.
db, err := database.Open(ctx, &shutdownWg, cfg.dbPath)
db, err := database.Open(ctx, &shutdownWg, cfg.dbPath, cfg.BackupInterval)
if err != nil {
log.Errorf("Database error: %v", err)
requestShutdown()