Add dcrd to VSP status.
This renames "Voting wallet status" to "VSP status" and it now includes the status of the local dcrd instance. This change impacts both the /admin page of the UI, and the response of the /admin/status API endpoint.
This commit is contained in:
parent
d337e0a321
commit
6acc5be10c
@ -71,7 +71,7 @@ func blockConnected() {
|
|||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
dcrdClient, err := dcrdRPC.Client(ctx, netParams)
|
dcrdClient, _, err := dcrdRPC.Client(ctx, netParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("%s: %v", funcName, err)
|
log.Errorf("%s: %v", funcName, err)
|
||||||
return
|
return
|
||||||
@ -314,7 +314,7 @@ func (n *NotificationHandler) Close() error {
|
|||||||
func connectNotifier(shutdownCtx context.Context, dcrdWithNotifs rpc.DcrdConnect) error {
|
func connectNotifier(shutdownCtx context.Context, dcrdWithNotifs rpc.DcrdConnect) error {
|
||||||
notifierClosed = make(chan struct{})
|
notifierClosed = make(chan struct{})
|
||||||
|
|
||||||
dcrdClient, err := dcrdWithNotifs.Client(shutdownCtx, netParams)
|
dcrdClient, _, err := dcrdWithNotifs.Client(shutdownCtx, netParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -411,7 +411,7 @@ func checkWalletConsistency() {
|
|||||||
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
|
||||||
dcrdClient, err := dcrdRPC.Client(ctx, netParams)
|
dcrdClient, _, err := dcrdRPC.Client(ctx, netParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("%s: %v", funcName, err)
|
log.Errorf("%s: %v", funcName, err)
|
||||||
return
|
return
|
||||||
|
|||||||
@ -436,7 +436,7 @@ func (vdb *VspDatabase) CheckIntegrity(ctx context.Context, params *chaincfg.Par
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
dcrdClient, err := dcrd.Client(ctx, params)
|
dcrdClient, _, err := dcrd.Client(ctx, params)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -118,13 +118,13 @@ The `[WRN]` level is used to indicate events which are of interest, but do not
|
|||||||
necessarily require investigation (eg. bad requests from clients, recoverable
|
necessarily require investigation (eg. bad requests from clients, recoverable
|
||||||
errors).
|
errors).
|
||||||
|
|
||||||
### Voting Wallet Status
|
### VSP Status
|
||||||
|
|
||||||
The current status of the voting wallets is displayed in a table on the `/admin`
|
The current status of the VSP is displayed in a table on the `/admin`
|
||||||
page, and the same information can be retrieved as a JSON object from
|
page, and the same information can be retrieved as a JSON object from
|
||||||
`/admin/status` for automated monitoring. This endpoint requires Basic HTTP
|
`/admin/status` for automated monitoring. This endpoint requires Basic HTTP
|
||||||
Authentication with the username `admin` and the password set in vspd
|
Authentication with the username `admin` and the password set in vspd
|
||||||
configuration. A 200 HTTP status will be returned if the voting wallets seem
|
configuration. A 200 HTTP status will be returned if the VSP seems
|
||||||
healthy, or a 500 status will be used to indicate something is wrong.
|
healthy, or a 500 status will be used to indicate something is wrong.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@ -133,16 +133,23 @@ $ curl --user admin:12345 --request GET http://localhost:8800/admin/status
|
|||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
"wss://127.0.0.1:20111/ws":
|
"dcrd": {
|
||||||
{
|
"host": "wss://127.0.0.1:19109/ws",
|
||||||
"connected":true,
|
"connected": true,
|
||||||
"infoerror":false,
|
"bestblockerror": false,
|
||||||
"daemonconnected":true,
|
"bestblockheight": 802572
|
||||||
"voteversion":8,
|
},
|
||||||
"unlocked":true,
|
"wallets": {
|
||||||
"voting":true,
|
"wss://127.0.0.1:20111/ws": {
|
||||||
"bestblockerror":false,
|
"connected": true,
|
||||||
"bestblockheight":462345
|
"infoerror": false,
|
||||||
|
"daemonconnected": true,
|
||||||
|
"voteversion": 10,
|
||||||
|
"unlocked": true,
|
||||||
|
"voting": true,
|
||||||
|
"bestblockerror": false,
|
||||||
|
"bestblockheight": 802572
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|||||||
22
rpc/dcrd.go
22
rpc/dcrd.go
@ -47,16 +47,16 @@ func SetupDcrd(user, pass, addr string, cert []byte, n wsrpc.Notifier) DcrdConne
|
|||||||
|
|
||||||
// Client creates a new DcrdRPC client instance. Returns an error if dialing
|
// Client creates a new DcrdRPC client instance. Returns an error if dialing
|
||||||
// dcrd fails or if dcrd is misconfigured.
|
// dcrd fails or if dcrd is misconfigured.
|
||||||
func (d *DcrdConnect) Client(ctx context.Context, netParams *chaincfg.Params) (*DcrdRPC, error) {
|
func (d *DcrdConnect) Client(ctx context.Context, netParams *chaincfg.Params) (*DcrdRPC, string, error) {
|
||||||
c, newConnection, err := d.dial(ctx)
|
c, newConnection, err := d.dial(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("dcrd connection error: %w", err)
|
return nil, d.addr, fmt.Errorf("dcrd connection error: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// If this is a reused connection, we don't need to validate the dcrd config
|
// If this is a reused connection, we don't need to validate the dcrd config
|
||||||
// again.
|
// again.
|
||||||
if !newConnection {
|
if !newConnection {
|
||||||
return &DcrdRPC{c, ctx}, nil
|
return &DcrdRPC{c, ctx}, d.addr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify dcrd is at the required api version.
|
// Verify dcrd is at the required api version.
|
||||||
@ -64,19 +64,19 @@ func (d *DcrdConnect) Client(ctx context.Context, netParams *chaincfg.Params) (*
|
|||||||
err = c.Call(ctx, "version", &verMap)
|
err = c.Call(ctx, "version", &verMap)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.Close()
|
d.Close()
|
||||||
return nil, fmt.Errorf("dcrd version check failed: %w", err)
|
return nil, d.addr, fmt.Errorf("dcrd version check failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ver, exists := verMap["dcrdjsonrpcapi"]
|
ver, exists := verMap["dcrdjsonrpcapi"]
|
||||||
if !exists {
|
if !exists {
|
||||||
d.Close()
|
d.Close()
|
||||||
return nil, fmt.Errorf("dcrd version response missing 'dcrdjsonrpcapi'")
|
return nil, d.addr, fmt.Errorf("dcrd version response missing 'dcrdjsonrpcapi'")
|
||||||
}
|
}
|
||||||
|
|
||||||
sVer := semver{ver.Major, ver.Minor, ver.Patch}
|
sVer := semver{ver.Major, ver.Minor, ver.Patch}
|
||||||
if !semverCompatible(requiredDcrdVersion, sVer) {
|
if !semverCompatible(requiredDcrdVersion, sVer) {
|
||||||
d.Close()
|
d.Close()
|
||||||
return nil, fmt.Errorf("dcrd has incompatible JSON-RPC version: got %s, expected %s",
|
return nil, d.addr, fmt.Errorf("dcrd has incompatible JSON-RPC version: got %s, expected %s",
|
||||||
sVer, requiredDcrdVersion)
|
sVer, requiredDcrdVersion)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -85,11 +85,11 @@ func (d *DcrdConnect) Client(ctx context.Context, netParams *chaincfg.Params) (*
|
|||||||
err = c.Call(ctx, "getcurrentnet", &netID)
|
err = c.Call(ctx, "getcurrentnet", &netID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.Close()
|
d.Close()
|
||||||
return nil, fmt.Errorf("dcrd getcurrentnet check failed: %w", err)
|
return nil, d.addr, fmt.Errorf("dcrd getcurrentnet check failed: %w", err)
|
||||||
}
|
}
|
||||||
if netID != netParams.Net {
|
if netID != netParams.Net {
|
||||||
d.Close()
|
d.Close()
|
||||||
return nil, fmt.Errorf("dcrd running on %s, expected %s", netID, netParams.Net)
|
return nil, d.addr, fmt.Errorf("dcrd running on %s, expected %s", netID, netParams.Net)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify dcrd has tx index enabled (required for getrawtransaction).
|
// Verify dcrd has tx index enabled (required for getrawtransaction).
|
||||||
@ -97,14 +97,14 @@ func (d *DcrdConnect) Client(ctx context.Context, netParams *chaincfg.Params) (*
|
|||||||
err = c.Call(ctx, "getinfo", &info)
|
err = c.Call(ctx, "getinfo", &info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.Close()
|
d.Close()
|
||||||
return nil, fmt.Errorf("dcrd getinfo check failed: %w", err)
|
return nil, d.addr, fmt.Errorf("dcrd getinfo check failed: %w", err)
|
||||||
}
|
}
|
||||||
if !info.TxIndex {
|
if !info.TxIndex {
|
||||||
d.Close()
|
d.Close()
|
||||||
return nil, errors.New("dcrd does not have transaction index enabled (--txindex)")
|
return nil, d.addr, errors.New("dcrd does not have transaction index enabled (--txindex)")
|
||||||
}
|
}
|
||||||
|
|
||||||
return &DcrdRPC{c, ctx}, nil
|
return &DcrdRPC{c, ctx}, d.addr, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetRawTransaction uses getrawtransaction RPC to retrieve details about the
|
// GetRawTransaction uses getrawtransaction RPC to retrieve details about the
|
||||||
|
|||||||
@ -27,6 +27,16 @@ type WalletStatus struct {
|
|||||||
BestBlockHeight int64 `json:"bestblockheight"`
|
BestBlockHeight int64 `json:"bestblockheight"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DcrdStatus describes the current status of the local instance of dcrd used by
|
||||||
|
// vspd. This is used by the admin.html template, and also serialized to JSON
|
||||||
|
// for the /admin/status endpoint.
|
||||||
|
type DcrdStatus struct {
|
||||||
|
Host string `json:"host"`
|
||||||
|
Connected bool `json:"connected"`
|
||||||
|
BestBlockError bool `json:"bestblockerror"`
|
||||||
|
BestBlockHeight uint32 `json:"bestblockheight"`
|
||||||
|
}
|
||||||
|
|
||||||
type searchResult struct {
|
type searchResult struct {
|
||||||
Hash string
|
Hash string
|
||||||
Found bool
|
Found bool
|
||||||
@ -36,6 +46,31 @@ type searchResult struct {
|
|||||||
MaxVoteChanges int
|
MaxVoteChanges int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func dcrdStatus(c *gin.Context) DcrdStatus {
|
||||||
|
hostname := c.MustGet("DcrdHostname").(string)
|
||||||
|
status := DcrdStatus{Host: hostname}
|
||||||
|
|
||||||
|
dcrdClient := c.MustGet("DcrdClient").(*rpc.DcrdRPC)
|
||||||
|
dcrdErr := c.MustGet("DcrdClientErr")
|
||||||
|
if dcrdErr != nil {
|
||||||
|
log.Errorf("could not get dcrd client: %v", dcrdErr.(error))
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
|
||||||
|
status.Connected = true
|
||||||
|
|
||||||
|
bestBlock, err := dcrdClient.GetBestBlockHeader()
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("could not get dcrd best block header: %v", err)
|
||||||
|
status.BestBlockError = true
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
|
||||||
|
status.BestBlockHeight = bestBlock.Height
|
||||||
|
|
||||||
|
return status
|
||||||
|
}
|
||||||
|
|
||||||
func walletStatus(c *gin.Context) map[string]WalletStatus {
|
func walletStatus(c *gin.Context) map[string]WalletStatus {
|
||||||
walletClients := c.MustGet("WalletClients").([]*rpc.WalletRPC)
|
walletClients := c.MustGet("WalletClients").([]*rpc.WalletRPC)
|
||||||
failedWalletClients := c.MustGet("FailedWalletClients").([]string)
|
failedWalletClients := c.MustGet("FailedWalletClients").([]string)
|
||||||
@ -92,7 +127,17 @@ func statusJSON(c *gin.Context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
c.AbortWithStatusJSON(httpStatus, wallets)
|
dcrd := dcrdStatus(c)
|
||||||
|
|
||||||
|
// Respond with HTTP status 500 if dcrd has issues.
|
||||||
|
if !dcrd.Connected || dcrd.BestBlockError {
|
||||||
|
httpStatus = http.StatusInternalServerError
|
||||||
|
}
|
||||||
|
|
||||||
|
c.AbortWithStatusJSON(httpStatus, gin.H{
|
||||||
|
"wallets": wallets,
|
||||||
|
"dcrd": dcrd,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// adminPage is the handler for "GET /admin".
|
// adminPage is the handler for "GET /admin".
|
||||||
@ -101,6 +146,7 @@ func adminPage(c *gin.Context) {
|
|||||||
"WebApiCache": getCache(),
|
"WebApiCache": getCache(),
|
||||||
"WebApiCfg": cfg,
|
"WebApiCfg": cfg,
|
||||||
"WalletStatus": walletStatus(c),
|
"WalletStatus": walletStatus(c),
|
||||||
|
"DcrdStatus": dcrdStatus(c),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -147,6 +193,7 @@ func ticketSearch(c *gin.Context) {
|
|||||||
"WebApiCache": getCache(),
|
"WebApiCache": getCache(),
|
||||||
"WebApiCfg": cfg,
|
"WebApiCfg": cfg,
|
||||||
"WalletStatus": walletStatus(c),
|
"WalletStatus": walletStatus(c),
|
||||||
|
"DcrdStatus": dcrdStatus(c),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -69,7 +69,7 @@ func updateCache(ctx context.Context, db *database.VspDatabase,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get latest best block height.
|
// Get latest best block height.
|
||||||
dcrdClient, err := dcrd.Client(ctx, netParams)
|
dcrdClient, _, err := dcrd.Client(ctx, netParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@ -75,10 +75,11 @@ func requireAdmin() gin.HandlerFunc {
|
|||||||
// downstream handlers to make use of.
|
// downstream handlers to make use of.
|
||||||
func withDcrdClient(dcrd rpc.DcrdConnect) gin.HandlerFunc {
|
func withDcrdClient(dcrd rpc.DcrdConnect) gin.HandlerFunc {
|
||||||
return func(c *gin.Context) {
|
return func(c *gin.Context) {
|
||||||
client, err := dcrd.Client(c, cfg.NetParams)
|
client, hostname, err := dcrd.Client(c, cfg.NetParams)
|
||||||
// Don't handle the error here, add it to the context and let downstream
|
// Don't handle the error here, add it to the context and let downstream
|
||||||
// handlers decide what to do with it.
|
// handlers decide what to do with it.
|
||||||
c.Set("DcrdClient", client)
|
c.Set("DcrdClient", client)
|
||||||
|
c.Set("DcrdHostname", hostname)
|
||||||
c.Set("DcrdClientErr", err)
|
c.Set("DcrdClientErr", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -151,34 +151,41 @@ footer .code {
|
|||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.vsp-status {
|
||||||
|
padding: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
#status-table th,
|
.vsp-status table {
|
||||||
#status-table td {
|
margin-bottom: 28px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.vsp-status table th,
|
||||||
|
.vsp-status table td {
|
||||||
border: 1px solid #edeff1;
|
border: 1px solid #edeff1;
|
||||||
vertical-align: middle;
|
vertical-align: middle;
|
||||||
text-align: center;
|
text-align: center;
|
||||||
}
|
}
|
||||||
|
|
||||||
#status-table .center {
|
.vsp-status table .center {
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: center;
|
justify-content: center;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
}
|
}
|
||||||
|
|
||||||
#status-table .status {
|
.vsp-status table .status {
|
||||||
height: 30px;
|
height: 30px;
|
||||||
padding-left: 30px;
|
padding-left: 30px;
|
||||||
}
|
}
|
||||||
|
|
||||||
#status-table .good {
|
.vsp-status table .good {
|
||||||
background: url(/public/images/success-icon.svg) no-repeat center center;
|
background: url(/public/images/success-icon.svg) no-repeat center center;
|
||||||
}
|
}
|
||||||
|
|
||||||
#status-table .bad {
|
.vsp-status table .bad {
|
||||||
background: url(/public/images/error-icon.svg) no-repeat left center;
|
background: url(/public/images/error-icon.svg) no-repeat left center;
|
||||||
}
|
}
|
||||||
|
|
||||||
#status-table .with-text {
|
.vsp-status table .with-text {
|
||||||
padding-left: 40px;
|
padding-left: 40px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -46,7 +46,7 @@
|
|||||||
hidden
|
hidden
|
||||||
>
|
>
|
||||||
<ul>
|
<ul>
|
||||||
<li><label for="tabset_1_1">Wallet Status</label></li>
|
<li><label for="tabset_1_1">VSP Status</label></li>
|
||||||
<li><label for="tabset_1_2">Ticket Search</label></li>
|
<li><label for="tabset_1_2">Ticket Search</label></li>
|
||||||
<li><label for="tabset_1_3">Database</label></li>
|
<li><label for="tabset_1_3">Database</label></li>
|
||||||
<li><label for="tabset_1_4">Logout</label></li>
|
<li><label for="tabset_1_4">Logout</label></li>
|
||||||
@ -54,15 +54,59 @@
|
|||||||
|
|
||||||
<div>
|
<div>
|
||||||
|
|
||||||
<section>
|
<section class="d-flex row justify-content-center">
|
||||||
<table id="status-table" class="w-100 mb-0">
|
|
||||||
|
<div class="vsp-status">
|
||||||
|
<h1>Local dcrd</h1>
|
||||||
|
|
||||||
|
<table class="vsp-status">
|
||||||
<thead>
|
<thead>
|
||||||
<th>URL</th>
|
<th>URL</th>
|
||||||
<th>Height</th>
|
<th>Height</th>
|
||||||
<th>Connected</th>
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>{{ stripWss .DcrdStatus.Host }}</td>
|
||||||
|
|
||||||
|
{{ if .DcrdStatus.Connected }}
|
||||||
|
|
||||||
|
{{ if .DcrdStatus.BestBlockError }}
|
||||||
|
<td>
|
||||||
|
<div class="center">
|
||||||
|
<div class="status bad center with-text">
|
||||||
|
Error
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
{{ else }}
|
||||||
|
<td>{{ .DcrdStatus.BestBlockHeight }}</td>
|
||||||
|
{{ end }}
|
||||||
|
|
||||||
|
{{else}}
|
||||||
|
<td>
|
||||||
|
<div class="center">
|
||||||
|
<div class="status bad center with-text">
|
||||||
|
Cannot connect
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
{{end}}
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="vsp-status">
|
||||||
|
<h1>Voting Wallets</h1>
|
||||||
|
|
||||||
|
<table class="vsp-status">
|
||||||
|
<thead>
|
||||||
|
<th>URL</th>
|
||||||
|
<th>Height</th>
|
||||||
|
<th>Connected<br />to dcrd</th>
|
||||||
<th>Unlocked</th>
|
<th>Unlocked</th>
|
||||||
<th>Voting</th>
|
<th>Voting</th>
|
||||||
<th>Vote Version</th>
|
<th>Vote<br />Version</th>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
{{ range $host, $status := .WalletStatus }}
|
{{ range $host, $status := .WalletStatus }}
|
||||||
@ -128,6 +172,8 @@
|
|||||||
{{end}}
|
{{end}}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section>
|
<section>
|
||||||
|
|||||||
@ -232,14 +232,14 @@ func router(debugMode bool, cookieSecret []byte, dcrd rpc.DcrdConnect, wallets r
|
|||||||
admin := router.Group("/admin").Use(
|
admin := router.Group("/admin").Use(
|
||||||
withWalletClients(wallets), withSession(cookieStore), requireAdmin(),
|
withWalletClients(wallets), withSession(cookieStore), requireAdmin(),
|
||||||
)
|
)
|
||||||
admin.GET("", adminPage)
|
admin.GET("", withDcrdClient(dcrd), adminPage)
|
||||||
admin.POST("/ticket", ticketSearch)
|
admin.POST("/ticket", withDcrdClient(dcrd), ticketSearch)
|
||||||
admin.GET("/backup", downloadDatabaseBackup)
|
admin.GET("/backup", downloadDatabaseBackup)
|
||||||
admin.POST("/logout", adminLogout)
|
admin.POST("/logout", adminLogout)
|
||||||
|
|
||||||
// Require Basic HTTP Auth on /admin/status endpoint.
|
// Require Basic HTTP Auth on /admin/status endpoint.
|
||||||
basic := router.Group("/admin").Use(
|
basic := router.Group("/admin").Use(
|
||||||
withWalletClients(wallets), gin.BasicAuth(gin.Accounts{
|
withDcrdClient(dcrd), withWalletClients(wallets), gin.BasicAuth(gin.Accounts{
|
||||||
"admin": cfg.AdminPass,
|
"admin": cfg.AdminPass,
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user