stakey/vspd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store records of vote choice changes

Browse Source
This commit is contained in:
jholdstock 2020-08-21 16:01:21 +01:00 committed by David Hill
parent d0c3abf258
commit 825a717ca7
15 changed files with 426 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
database/database.go
View File

@ -24,6 +24,7 @@ import (
// convenience functions.
type VspDatabase struct {
db *bolt.DB
maxVoteChangeRecords int
ticketsMtx sync.RWMutex
}
@ -35,6 +36,8 @@ var (
vspBktK = []byte("vspbkt")
// ticketbkt stores all tickets known by this VSP.
ticketBktK = []byte("ticketbkt")
// votechangebkt stores records of web requests which update vote choices.
voteChangeBktK = []byte("votechangebkt")
// version is the current database version.
versionK = []byte("version")
// feeXPub is the extended public key used for collecting VSP fees.
@ -150,6 +153,12 @@ func CreateNew(dbFile, feeXPub string) error {
return fmt.Errorf("failed to create %s bucket: %v", string(ticketBktK), err)
}
// Create vote change bucket.
_, err = vspBkt.CreateBucket(voteChangeBktK)
if err != nil {
return fmt.Errorf("failed to create %s bucket: %v", string(voteChangeBktK), err)
}
return nil
})
@ -164,7 +173,7 @@ func CreateNew(dbFile, feeXPub string) error {
// Open initializes and returns an open database. An error is returned if no
// database file is found at the provided path.
func Open(ctx context.Context, shutdownWg *sync.WaitGroup, dbFile string, backupInterval time.Duration) (*VspDatabase, error) {
func Open(ctx context.Context, shutdownWg *sync.WaitGroup, dbFile string, backupInterval time.Duration, maxVoteChangeRecords int) (*VspDatabase, error) {
// Error if db file does not exist. This is needed because bolt.Open will
// silently create a new empty database if the file does not exist. A new
@ -200,7 +209,7 @@ func Open(ctx context.Context, shutdownWg *sync.WaitGroup, dbFile string, backup
}
}()
return &VspDatabase{db: db}, nil
return &VspDatabase{db: db, maxVoteChangeRecords: maxVoteChangeRecords}, nil
}
// Close will close the database and then make a copy of the database to the

4
database/database_test.go
View File

@ -16,6 +16,7 @@ var (
testDb = "test.db"
backupDb = "test.db-backup"
db *VspDatabase
maxVoteChangeRecords = 3
)
// TestDatabase runs all database tests.
@ -33,6 +34,7 @@ func TestDatabase(t *testing.T) {
"testFilterTickets": testFilterTickets,
"testAddressIndex": testAddressIndex,
"testDeleteTicket": testDeleteTicket,
"testVoteChangeRecords": testVoteChangeRecords,
}
for testName, test := range tests {
@ -44,7 +46,7 @@ func TestDatabase(t *testing.T) {
if err != nil {
t.Fatalf("error creating test database: %v", err)
}
db, err = Open(ctx, &wg, testDb, time.Hour)
db, err = Open(ctx, &wg, testDb, time.Hour, maxVoteChangeRecords)
if err != nil {
t.Fatalf("error opening test database: %v", err)
}

1
database/ticket.go
View File

@ -139,6 +139,7 @@ func (vdb *VspDatabase) DeleteTicket(ticket Ticket) error {
return nil
})
}
func (vdb *VspDatabase) UpdateTicket(ticket Ticket) error {
defer vdb.ticketsMtx.Unlock()
vdb.ticketsMtx.Lock()

131
database/votechange.go Normal file
View File

@ -0,0 +1,131 @@
// Copyright (c) 2020 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.
package database
import (
"encoding/binary"
"encoding/json"
"fmt"
"math"
bolt "go.etcd.io/bbolt"
)
// VoteChangeRecord is serialized to json and stored in bbolt db. The json keys
// are deliberately kept short because they are duplicated many times in the db.
type VoteChangeRecord struct {
Request string `json:"req"`
RequestSignature string `json:"reqs"`
Response string `json:"rsp"`
ResponseSignature string `json:"rsps"`
}
// SaveVoteChange will insert the provided vote change record into the database,
// and if this breaches the maximum amount of allowed records, delete the oldest
// one which is currently stored.
func (vdb *VspDatabase) SaveVoteChange(ticketHash string, record VoteChangeRecord) error {
return vdb.db.Update(func(tx *bolt.Tx) error {
// Create or get a bucket for this ticket.
bkt, err := tx.Bucket(vspBktK).Bucket(voteChangeBktK).
CreateBucketIfNotExists([]byte(ticketHash))
if err != nil {
return fmt.Errorf("failed to create vote change bucket (ticketHash=%s): %v",
ticketHash, err)
}
// Serialize record for storage in the database.
recordBytes, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("could not marshal vote change record: %v", err)
}
// Records are stored using a serially increasing integer as the key.
// Loop through the bucket to count the records, as well as finding the
// most recent and the oldest record.
var count int
highest := uint32(0)
lowest := uint32(math.MaxUint32)
err = bkt.ForEach(func(k, v []byte) error {
count++
key := binary.LittleEndian.Uint32(k)
if key > highest {
highest = key
}
if key < lowest {
lowest = key
}
return nil
})
if err != nil {
return fmt.Errorf("error iterating over vote change bucket: %v", err)
}
// If bucket is at (or over) the limit of max allowed records, remove
// the oldest one.
if count >= vdb.maxVoteChangeRecords {
keyBytes := make([]byte, 4)
binary.LittleEndian.PutUint32(keyBytes, lowest)
err = bkt.Delete(keyBytes)
if err != nil {
return fmt.Errorf("failed to delete old vote change record: %v", err)
}
}
// Insert record with index 0 if the bucket is currently empty,
// otherwise use most recent + 1.
var newKey uint32
if count > 0 {
newKey = highest + 1
}
keyBytes := make([]byte, 4)
binary.LittleEndian.PutUint32(keyBytes, newKey)
// Insert record.
err = bkt.Put(keyBytes, recordBytes)
if err != nil {
return fmt.Errorf("could not store vote change record: %v", err)
}
return nil
})
}
// GetVoteChanges retrieves all of the stored vote change records for the
// provided ticket hash.
func (vdb *VspDatabase) GetVoteChanges(ticketHash string) (map[uint32]VoteChangeRecord, error) {
records := make(map[uint32]VoteChangeRecord)
err := vdb.db.View(func(tx *bolt.Tx) error {
bkt := tx.Bucket(vspBktK).Bucket(voteChangeBktK).
Bucket([]byte(ticketHash))
if bkt == nil {
return nil
}
err := bkt.ForEach(func(k, v []byte) error {
var record VoteChangeRecord
err := json.Unmarshal(v, &record)
if err != nil {
return fmt.Errorf("could not unmarshal vote change record: %v", err)
}
records[binary.LittleEndian.Uint32(k)] = record
return nil
})
if err != nil {
return fmt.Errorf("error iterating over vote change bucket: %v", err)
}
return nil
})
return records, err
}

66
database/votechange_test.go Normal file
View File

@ -0,0 +1,66 @@
// Copyright (c) 2020 The Decred developers
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.
package database
import (
"testing"
)
func exampleRecord() VoteChangeRecord {
return VoteChangeRecord{
Request: "Request",
RequestSignature: "RequestSignature",
Response: "Response",
ResponseSignature: "ResponseSignature",
}
}
func testVoteChangeRecords(t *testing.T) {
hash := "MyHash"
record := exampleRecord()
// Insert a record into the database.
err := db.SaveVoteChange(hash, record)
if err != nil {
t.Fatalf("error storing vote change record in database: %v", err)
}
// Retrieve record and check values.
retrieved, err := db.GetVoteChanges(hash)
if err != nil {
t.Fatalf("error retrieving vote change records: %v", err)
}
if len(retrieved) != 1 ||
retrieved[0].Request != record.Request ||
retrieved[0].RequestSignature != record.RequestSignature ||
retrieved[0].Response != record.Response ||
retrieved[0].ResponseSignature != record.ResponseSignature {
t.Fatal("retrieved record didnt match expected")
}
// Insert some more records, giving us one greater than the limit.
for i := 0; i < maxVoteChangeRecords; i++ {
err = db.SaveVoteChange(hash, record)
if err != nil {
t.Fatalf("error storing vote change record in database: %v", err)
}
}
// Retrieve records.
retrieved, err = db.GetVoteChanges(hash)
if err != nil {
t.Fatalf("error retrieving vote change records: %v", err)
}
// Oldest record should have been deleted.
if len(retrieved) != maxVoteChangeRecords {
t.Fatalf("vote change record limit breached")
}
if _, ok := retrieved[0]; ok {
t.Fatalf("oldest vote change record should have been deleted")
}
}

9
vspd.go
View File

@ -19,6 +19,12 @@ import (
"github.com/decred/vspd/webapi"
)
// maxVoteChangeRecords defines how many vote change records will be stored for
// each ticket. The limit is in place to mitigate DoS attacks on server storage
// space. When storing a new record breaches this limit, the oldest record in
// the database is deleted.
const maxVoteChangeRecords = 10
func main() {
// Create a context that is cancelled when a shutdown request is received
// through an interrupt signal.
@ -59,7 +65,7 @@ func run(ctx context.Context) error {
defer log.Info("Shutdown complete")
// Open database.
db, err := database.Open(ctx, &shutdownWg, cfg.dbPath, cfg.BackupInterval)
db, err := database.Open(ctx, &shutdownWg, cfg.dbPath, cfg.BackupInterval, maxVoteChangeRecords)
if err != nil {
log.Errorf("Database error: %v", err)
requestShutdown()
@ -86,6 +92,7 @@ func run(ctx context.Context) error {
AdminPass: cfg.AdminPass,
Debug: cfg.WebServerDebug,
Designation: cfg.Designation,
MaxVoteChangeRecords: maxVoteChangeRecords,
}
err = webapi.Start(ctx, shutdownRequestChannel, &shutdownWg, cfg.Listen, db,
dcrd, wallets, apiCfg)

11
webapi/admin.go
View File

@ -100,16 +100,25 @@ func ticketSearch(c *gin.Context) {
ticket, found, err := db.GetTicketByHash(hash)
if err != nil {
log.Errorf("db.GetTicketByHash error: %v", err)
log.Errorf("db.GetTicketByHash error (ticketHash=%s): %v", hash, err)
c.String(http.StatusInternalServerError, "Error getting ticket from db")
return
}
voteChanges, err := db.GetVoteChanges(hash)
if err != nil {
log.Errorf("db.GetVoteChanges error (ticketHash=%s): %v", hash, err)
c.String(http.StatusInternalServerError, "Error getting vote changes from db")
return
}
c.HTML(http.StatusOK, "admin.html", gin.H{
"SearchResult": gin.H{
"Hash": hash,
"Found": found,
"Ticket": ticket,
"VoteChanges": voteChanges,
"MaxVoteChanges": cfg.MaxVoteChangeRecords,
},
"VspStats": getVSPStats(),
"WalletStatus": walletStatus(c),

4
webapi/getfeeaddress.go
View File

@ -13,6 +13,7 @@ import (
"github.com/decred/vspd/database"
"github.com/decred/vspd/rpc"
"github.com/gin-gonic/gin"
"github.com/gin-gonic/gin/binding"
)
// addrMtx protects getNewFeeAddress.
@ -71,6 +72,7 @@ func feeAddress(c *gin.Context) {
knownTicket := c.MustGet("KnownTicket").(bool)
commitmentAddress := c.MustGet("CommitmentAddress").(string)
dcrdClient := c.MustGet("DcrdClient").(*rpc.DcrdRPC)
reqBytes := c.MustGet("RequestBytes").([]byte)
if cfg.VspClosed {
sendError(errVspClosed, c)
@ -78,7 +80,7 @@ func feeAddress(c *gin.Context) {
}
var request feeAddressRequest
if err := c.ShouldBindJSON(&request); err != nil {
if err := binding.JSON.BindBody(reqBytes, &request); err != nil {
log.Warnf("%s: Bad request (clientIP=%s): %v", funcName, c.ClientIP(), err)
sendErrorWithMsg(err.Error(), errBadRequest, c)
return

9
webapi/middleware.go
View File

@ -203,16 +203,17 @@ func vspAuth() gin.HandlerFunc {
return func(c *gin.Context) {
const funcName = "vspAuth"
// Read request bytes and then replace the request reader for
// downstream handlers to use.
// Read request bytes.
reqBytes, err := ioutil.ReadAll(c.Request.Body)
if err != nil {
log.Warnf("%s: Error reading request (clientIP=%s): %v", funcName, c.ClientIP(), err)
sendErrorWithMsg(err.Error(), errBadRequest, c)
return
}
c.Request.Body.Close()
c.Request.Body = ioutil.NopCloser(bytes.NewBuffer(reqBytes))
// Add request bytes to request context for downstream handlers to reuse.
// Necessary because the request body reader can only be used once.
c.Set("RequestBytes", reqBytes)
// Parse request and ensure there is a ticket hash included.
var request ticketHashRequest

20
webapi/payfee.go
View File

@ -14,6 +14,7 @@ import (
"github.com/decred/vspd/database"
"github.com/decred/vspd/rpc"
"github.com/gin-gonic/gin"
"github.com/gin-gonic/gin/binding"
)
// payFee is the handler for "POST /api/v3/payfee".
@ -24,6 +25,7 @@ func payFee(c *gin.Context) {
ticket := c.MustGet("Ticket").(database.Ticket)
knownTicket := c.MustGet("KnownTicket").(bool)
dcrdClient := c.MustGet("DcrdClient").(*rpc.DcrdRPC)
reqBytes := c.MustGet("RequestBytes").([]byte)
if cfg.VspClosed {
sendError(errVspClosed, c)
@ -37,7 +39,7 @@ func payFee(c *gin.Context) {
}
var request payFeeRequest
if err := c.ShouldBindJSON(&request); err != nil {
if err := binding.JSON.BindBody(reqBytes, &request); err != nil {
log.Warnf("%s: Bad request (clientIP=%s): %v", funcName, c.ClientIP(), err)
sendErrorWithMsg(err.Error(), errBadRequest, c)
return
@ -257,8 +259,22 @@ findAddress:
funcName, ticket.Hash, ticket.FeeTxHash)
}
sendJSONResponse(payFeeResponse{
// Send success response to client.
resp, respSig := sendJSONResponse(payFeeResponse{
Timestamp: time.Now().Unix(),
Request: request,
}, c)
// Store a record of the vote choice change.
err = db.SaveVoteChange(
ticket.Hash,
database.VoteChangeRecord{
Request: string(reqBytes),
RequestSignature: c.GetHeader("VSP-Client-Signature"),
Response: resp,
ResponseSignature: respSig,
})
if err != nil {
log.Errorf("%s: Failed to store vote change record (ticketHash=%s): %v", err)
}
}

1
webapi/public/css/vspd.css
View File

@ -132,6 +132,7 @@ td.status-bad{
text-align: right;
}
.ticket-table td {
font-size: 14px;
text-align: left;
}

32
webapi/setvotechoices.go
View File

@ -5,11 +5,13 @@
package webapi
import (
"fmt"
"time"
"github.com/decred/vspd/database"
"github.com/decred/vspd/rpc"
"github.com/gin-gonic/gin"
"github.com/gin-gonic/gin/binding"
)
// setVoteChoices is the handler for "POST /api/v3/setvotechoices".
@ -20,6 +22,7 @@ func setVoteChoices(c *gin.Context) {
ticket := c.MustGet("Ticket").(database.Ticket)
knownTicket := c.MustGet("KnownTicket").(bool)
walletClients := c.MustGet("WalletClients").([]*rpc.WalletRPC)
reqBytes := c.MustGet("RequestBytes").([]byte)
// If we cannot set the vote choices on at least one voting wallet right
// now, don't update the database, just return an error.
@ -41,8 +44,17 @@ func setVoteChoices(c *gin.Context) {
return
}
// Only allow vote choices to be updated for live/immature tickets.
if ticket.Outcome != "" {
log.Warnf("%s: Ticket not eligible to vote (clientIP=%s, ticketHash=%s)",
funcName, c.ClientIP(), ticket.Hash)
sendErrorWithMsg(fmt.Sprintf("ticket not eligible to vote (status=%s)", ticket.Outcome),
errTicketCannotVote, c)
return
}
var request setVoteChoicesRequest
if err := c.ShouldBindJSON(&request); err != nil {
if err := binding.JSON.BindBody(reqBytes, &request); err != nil {
log.Warnf("%s: Bad request (clientIP=%s): %v", funcName, c.ClientIP(), err)
sendErrorWithMsg(err.Error(), errBadRequest, c)
return
@ -88,10 +100,22 @@ func setVoteChoices(c *gin.Context) {
// TODO: DB - error if given timestamp is older than any previous requests
// TODO: DB - store setvotechoices receipt in log
sendJSONResponse(setVoteChoicesResponse{
// Send success response to client.
resp, respSig := sendJSONResponse(setVoteChoicesResponse{
Timestamp: time.Now().Unix(),
Request: request,
}, c)
// Store a record of the vote choice change.
err = db.SaveVoteChange(
ticket.Hash,
database.VoteChangeRecord{
Request: string(reqBytes),
RequestSignature: c.GetHeader("VSP-Client-Signature"),
Response: resp,
ResponseSignature: respSig,
})
if err != nil {
log.Errorf("%s: Failed to store vote change record (ticketHash=%s): %v", err)
}
}

72
webapi/templates/admin.html
View File

@ -86,73 +86,109 @@
<div class="block__content">
<h1>Ticket Search</h1>
<form class="my-2" action="/admin/ticket" method="post">
<form class="mt-2 mb-4" action="/admin/ticket" method="post">
<input type="text" name="hash" size="64" minlength="64" maxlength="64" required placeholder="Ticket hash" autocomplete="off">
<button class="ml-3 btn btn-primary" type="submit">Search</button>
</form>
{{ with .SearchResult }}
{{ if .Found }}
{{ with .Ticket }}
<table class="table ticket-table mt-4 mb-0">
<h1>Search Result</h1>
<table class="table ticket-table mt-2 mb-4">
<tr>
<th>Hash</th>
<td>{{ .Hash }}</td>
<td>{{ .Ticket.Hash }}</td>
</tr>
<tr>
<th>Commitment Address</th>
<td>{{ .CommitmentAddress }}</td>
<td>{{ .Ticket.CommitmentAddress }}</td>
</tr>
<tr>
<th>Fee Address Index</th>
<td>{{ .FeeAddressIndex }}</td>
<td>{{ .Ticket.FeeAddressIndex }}</td>
</tr>
<tr>
<th>Fee Address</th>
<td>{{ .FeeAddress }}</td>
<td>{{ .Ticket.FeeAddress }}</td>
</tr>
<tr>
<th>Fee Amount</th>
<td>{{ .FeeAmount }} atoms</td>
<td>{{ .Ticket.FeeAmount }} atoms</td>
</tr>
<tr>
<th>Fee Expiration</th>
<td>{{ .FeeExpiration }}</td>
<td>{{ .Ticket.FeeExpiration }}</td>
</tr>
<tr>
<th>Confirmed</th>
<td>{{ .Confirmed }}</td>
<td>{{ .Ticket.Confirmed }}</td>
</tr>
<tr>
<th>Vote Choices</th>
<th>Current Vote Choices</th>
<td>
{{ range $key, $value := .VoteChoices }}
{{ range $key, $value := .Ticket.VoteChoices }}
{{ $key }}: {{ $value }} <br />
{{ end }}
</td>
</tr>
<tr>
<th>
Vote Choice Changes<br />
<em>({{ .MaxVoteChanges }} most recent)</em>
</th>
<td>
{{ range $key, $value := .VoteChanges }}
<details>
<summary>
{{ if eq $key 0}}
Initial choices
{{ else }}
Change {{ $key }}
{{ end }}
</summary>
<table class="table ticket-table my-2">
<tr>
<th>Request</th>
<td>{{ $value.Request }}</td>
</tr>
<tr>
<th>Request<br />Signature</th>
<td>{{ $value.RequestSignature }}</td>
</tr>
<tr>
<th>Response</th>
<td>{{ $value.Response }}</td>
</tr>
<tr>
<th>Response<br />Signature</th>
<td>{{ $value.ResponseSignature }}</td>
</tr>
</table>
</details>
{{end}}
</td>
</tr>
<tr>
<th>Voting WIF</th>
<td>{{ .VotingWIF }}</td>
<td>{{ .Ticket.VotingWIF }}</td>
</tr>
<tr>
<th>Fee Tx</th>
<td>{{ .FeeTxHex }}</td>
<td>{{ .Ticket.FeeTxHex }}</td>
</tr>
<tr>
<th>Fee Tx Hash</th>
<td>{{ .FeeTxHash }}</td>
<td>{{ .Ticket.FeeTxHash }}</td>
</tr>
<tr>
<th>Fee Tx Status</th>
<td>{{ .FeeTxStatus }}</td>
<td>{{ .Ticket.FeeTxStatus }}</td>
</tr>
<tr>
<th>Ticket Outcome</th>
<td>{{ .Outcome }}</td>
<td>{{ .Ticket.Outcome }}</td>
</tr>
</table>
{{ end }}
{{ else }}
<p>No ticket found with hash <span class="code">{{ .Hash }}</span></p>
{{ end }}

4
webapi/ticketstatus.go
View File

@ -9,6 +9,7 @@ import (
"github.com/decred/vspd/database"
"github.com/gin-gonic/gin"
"github.com/gin-gonic/gin/binding"
)
// ticketStatus is the handler for "POST /api/v3/ticketstatus".
@ -18,6 +19,7 @@ func ticketStatus(c *gin.Context) {
// Get values which have been added to context by middleware.
ticket := c.MustGet("Ticket").(database.Ticket)
knownTicket := c.MustGet("KnownTicket").(bool)
reqBytes := c.MustGet("RequestBytes").([]byte)
if !knownTicket {
log.Warnf("%s: Unknown ticket (clientIP=%s)", funcName, c.ClientIP())
@ -26,7 +28,7 @@ func ticketStatus(c *gin.Context) {
}
var request ticketStatusRequest
if err := c.ShouldBindJSON(&request); err != nil {
if err := binding.JSON.BindBody(reqBytes, &request); err != nil {
log.Warnf("%s: Bad request (clientIP=%s): %v", funcName, c.ClientIP(), err)
sendErrorWithMsg(err.Error(), errBadRequest, c)
return

17
webapi/webapi.go
View File

@ -31,6 +31,7 @@ type Config struct {
AdminPass string
Debug bool
Designation string
MaxVoteChangeRecords int
}
const (
@ -227,28 +228,34 @@ func router(debugMode bool, cookieSecret []byte, dcrd rpc.DcrdConnect, wallets r
return router
}
func sendJSONResponse(resp interface{}, c *gin.Context) {
// sendJSONResponse serializes the provided response, signs it, and sends the
// response to the client with a 200 OK status. Returns the seralized response
// and the signature.
func sendJSONResponse(resp interface{}, c *gin.Context) (string, string) {
dec, err := json.Marshal(resp)
if err != nil {
log.Errorf("JSON marshal error: %v", err)
sendError(errInternalError, c)
return
return "", ""
}
sig := ed25519.Sign(signPrivKey, dec)
c.Writer.Header().Set("VSP-Server-Signature", base64.StdEncoding.EncodeToString(sig))
sigStr := base64.StdEncoding.EncodeToString(sig)
c.Writer.Header().Set("VSP-Server-Signature", sigStr)
c.AbortWithStatusJSON(http.StatusOK, resp)
return string(dec), sigStr
}
// sendError returns an error response to the client using the default error
// sendError sends an error response to the client using the default error
// message.
func sendError(e apiError, c *gin.Context) {
msg := e.defaultMessage()
sendErrorWithMsg(msg, e, c)
}
// sendErrorWithMsg returns an error response to the client using the provided
// sendErrorWithMsg sends an error response to the client using the provided
// error message.
func sendErrorWithMsg(msg string, e apiError, c *gin.Context) {
status := e.httpStatus()