Skip to content

Commit 53ad4ae

Browse files
author
Dov Alperin
committed
Deregister standbys after x mins of inactivity
1 parent c8c7066 commit 53ad4ae

File tree

7 files changed

+102
-10
lines changed

7 files changed

+102
-10
lines changed

‎Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ WORKDIR /go/src/github.com/fly-examples/fly-postgres
77
COPY . .
88

99
RUN CGO_ENABLED=0 GOOS=linux go build -v -o /fly/bin/event_handler ./cmd/event_handler
10+
RUN CGO_ENABLED=0 GOOS=linux go build -v -o /fly/bin/standby_cleaner ./cmd/standby_cleaner
1011
RUN CGO_ENABLED=0 GOOS=linux go build -v -o /fly/bin/start ./cmd/start
1112
COPY ./bin/* /fly/bin/
1213

‎cmd/standby_cleaner/main.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"github.com/fly-apps/postgres-flex/pkg/flypg"
7+
"os"
8+
"time"
9+
)
10+
11+
var Minute int64 = 60
12+
13+
func main() {
14+
ctx := context.Background()
15+
flypgNode, err := flypg.NewNode()
16+
if err != nil {
17+
fmt.Printf("failed to reference node: %s\n", err)
18+
os.Exit(1)
19+
}
20+
21+
conn, err := flypgNode.RepMgr.NewLocalConnection(ctx)
22+
if err != nil {
23+
fmt.Printf("failed to open local connection: %s\n", err)
24+
os.Exit(1)
25+
}
26+
27+
ticker := time.NewTicker(5 * time.Second)
28+
defer ticker.Stop()
29+
30+
seenAt := map[int]int64{}
31+
32+
for _ = range ticker.C {
33+
role, err := flypgNode.RepMgr.CurrentRole(ctx, conn)
34+
if err != nil {
35+
fmt.Printf("Failed to check role: %s", err)
36+
continue
37+
}
38+
if role != "primary" {
39+
continue
40+
}
41+
standbys, err := flypgNode.RepMgr.Standbys(ctx, conn)
42+
if err != nil {
43+
fmt.Printf("Failed to get standbys: %s", err)
44+
continue
45+
}
46+
for _, standby := range standbys {
47+
newConn, err := flypgNode.RepMgr.NewRemoteConnection(ctx, standby.Ip)
48+
if err != nil {
49+
if time.Now().Unix()-seenAt[standby.Id] >= 10*Minute {
50+
err := flypg.RunCommand(fmt.Sprintf("repmgr standby unregister -f %s --node-id=%d", flypgNode.RepMgr.ConfigPath, standby.Id))
51+
if err != nil {
52+
fmt.Printf("Failed to deregister %d: %s", standby.Id, err)
53+
continue
54+
}
55+
delete(seenAt, standby.Id)
56+
}
57+
} else {
58+
seenAt[standby.Id] = time.Now().Unix()
59+
newConn.Close(ctx)
60+
}
61+
}
62+
}
63+
}

‎cmd/start/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ func main() {
5252
svisor.AddProcess("repmgrd", fmt.Sprintf("gosu postgres repmgrd -f %s --daemonize=false", node.RepMgr.ConfigPath),
5353
supervisor.WithRestart(0, 5*time.Second),
5454
)
55+
svisor.AddProcess("standby_cleaner", "/usr/local/bin/standby_cleaner", supervisor.WithRestart(0, 5*time.Second))
5556

5657
exporterEnv := map[string]string{
5758
"DATA_SOURCE_URI": fmt.Sprintf("[%s]:%d/postgres?sslmode=disable", node.PrivateIP, node.Port),

‎pkg/flypg/config.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ func (c *Config) Print(w io.Writer) error {
146146
}
147147

148148
func (c Config) EnableCustomConfig() error {
149-
if err := runCommand(fmt.Sprintf("touch %s", c.customConfigFilePath)); err != nil {
149+
if err := RunCommand(fmt.Sprintf("touch %s", c.customConfigFilePath)); err != nil {
150150
return err
151151
}
152152

‎pkg/flypg/node.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -491,7 +491,7 @@ func setDirOwnership() error {
491491
return err
492492
}
493493

494-
func runCommand(cmdStr string) error {
494+
func RunCommand(cmdStr string) error {
495495
pgUser, err := user.Lookup("postgres")
496496
if err != nil {
497497
return err

‎pkg/flypg/pgbouncer.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,15 @@ func (p *PGBouncer) ConfigurePrimary(ctx context.Context, primary string, reload
4141

4242
func (p *PGBouncer) initialize() error {
4343
cmdStr := fmt.Sprintf("mkdir -p %s", p.ConfigPath)
44-
if err := runCommand(cmdStr); err != nil {
44+
if err := RunCommand(cmdStr); err != nil {
4545
return err
4646
}
4747

4848
// If pgbouncer.ini file is not present, set defaults.
4949
if _, err := os.Stat(fmt.Sprintf("%s/pgbouncer.ini", p.ConfigPath)); err != nil {
5050
if os.IsNotExist(err) {
5151
cmdStr := fmt.Sprintf("cp /fly/pgbouncer.ini %s", p.ConfigPath)
52-
if err := runCommand(cmdStr); err != nil {
52+
if err := RunCommand(cmdStr); err != nil {
5353
return err
5454
}
5555
} else {

‎pkg/flypg/repmgr.go‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,10 @@ func (r *RepMgr) CurrentRole(ctx context.Context, pg *pgx.Conn) (string, error)
7474
return r.memberRole(ctx, pg, int(r.ID))
7575
}
7676

77+
func (r *RepMgr) Standbys(ctx context.Context, pg *pgx.Conn) ([]Standby, error) {
78+
return r.standbyStatuses(ctx, pg, int(r.ID))
79+
}
80+
7781
func (r *RepMgr) writeManagerConf() error {
7882
file, err := os.OpenFile(r.ConfigPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
7983
if err != nil {
@@ -110,7 +114,7 @@ func (r *RepMgr) writeManagerConf() error {
110114

111115
func (r *RepMgr) registerPrimary() error {
112116
cmdStr := fmt.Sprintf("repmgr -f %s primary register -F -v", r.ConfigPath)
113-
if err := runCommand(cmdStr); err != nil {
117+
if err := RunCommand(cmdStr); err != nil {
114118
return err
115119
}
116120

@@ -119,7 +123,7 @@ func (r *RepMgr) registerPrimary() error {
119123

120124
func (r *RepMgr) unregisterPrimary() error {
121125
cmdStr := fmt.Sprintf("repmgr -f %s primary unregister", r.ConfigPath)
122-
if err := runCommand(cmdStr); err != nil {
126+
if err := RunCommand(cmdStr); err != nil {
123127
return err
124128
}
125129

@@ -128,7 +132,7 @@ func (r *RepMgr) unregisterPrimary() error {
128132

129133
func (r *RepMgr) followPrimary() error {
130134
cmdStr := fmt.Sprintf("repmgr -f %s standby follow", r.ConfigPath)
131-
if err := runCommand(cmdStr); err != nil {
135+
if err := RunCommand(cmdStr); err != nil {
132136
fmt.Printf("failed to register standby: %s", err)
133137
}
134138

@@ -138,7 +142,7 @@ func (r *RepMgr) followPrimary() error {
138142
func (r *RepMgr) registerStandby() error {
139143
// Force re-registry to ensure the standby picks up any new configuration changes.
140144
cmdStr := fmt.Sprintf("repmgr -f %s standby register -F", r.ConfigPath)
141-
if err := runCommand(cmdStr); err != nil {
145+
if err := RunCommand(cmdStr); err != nil {
142146
fmt.Printf("failed to register standby: %s", err)
143147
}
144148

@@ -147,7 +151,7 @@ func (r *RepMgr) registerStandby() error {
147151

148152
func (r *RepMgr) clonePrimary(ipStr string) error {
149153
cmdStr := fmt.Sprintf("mkdir -p %s", r.DataDir)
150-
if err := runCommand(cmdStr); err != nil {
154+
if err := RunCommand(cmdStr); err != nil {
151155
return err
152156
}
153157

@@ -159,7 +163,7 @@ func (r *RepMgr) clonePrimary(ipStr string) error {
159163
r.ConfigPath)
160164

161165
fmt.Println(cmdStr)
162-
return runCommand(cmdStr)
166+
return RunCommand(cmdStr)
163167
}
164168

165169
func (r *RepMgr) writePasswdConf() error {
@@ -184,6 +188,29 @@ func (r *RepMgr) writePasswdConf() error {
184188
return nil
185189
}
186190

191+
type Standby struct {
192+
Id int
193+
Ip string
194+
}
195+
196+
func (r *RepMgr) standbyStatuses(ctx context.Context, pg *pgx.Conn, id int) ([]Standby, error) {
197+
sql := fmt.Sprintf("select node_id, node_name from repmgr.show_nodes where type = 'standby' and upstream_node_id = '%d';", id)
198+
var standbys []Standby
199+
rows, err := pg.Query(ctx, sql)
200+
if err != nil {
201+
return nil, err
202+
}
203+
for rows.Next() {
204+
var s Standby
205+
err := rows.Scan(&s.Id, &s.Ip)
206+
if err != nil {
207+
return nil, err
208+
}
209+
standbys = append(standbys, s)
210+
}
211+
return standbys, nil
212+
}
213+
187214
func (r *RepMgr) memberRole(ctx context.Context, pg *pgx.Conn, id int) (string, error) {
188215
sql := fmt.Sprintf("select n.type from repmgr.nodes n LEFT JOIN repmgr.nodes un ON un.node_id = n.upstream_node_id WHERE n.node_id = '%d';", id)
189216
var role string

0 commit comments

Comments
 (0)