Improve and actualize orchestrator-agent , extract many LVM commands into configuration file

.gitignore

@@ -2,4 +2,6 @@
 .vendor/
 .gopath/
 .idea/
+*.deb
+*.pcap
 *.log

README.md

@@ -96,6 +96,13 @@ The following is a complete list of configuration parameters:
 * `SnapshotMountPoint`                 (string), a known mountpoint onto which a `mount` command will mount snapshot volumes
 * `ContinuousPollSeconds`              (uint), internal clocking interval (default 60 seconds)
 * `ResubmitAgentIntervalMinutes`       (uint), interval at which the agent re-submits itself to *orchestrator* daemon
+* `LogicalVolumesCommand`              (string), command which list logical volumes, default implementation used lvs
+* `GetMountCommand`                    (string), command which get mount point parameters by mount point name, default implementation over cat /etc/mtab
+* `UnmountCommand`                     (string), command which unmount current mount point, default implementation just execute umount
+* `MountLVCommand`                     (string), command which mount selected snapshot, default implementation execute mount selected LVM snapshot 
+* `RemoveLVCommand`                    (string), command which remove selected snapshot, default implementation execute lvremove selected LVM snapshot
+* `MySQLTailErrorLogCommand`           (string), command which return last 20 lines from MySQL @@log_error file
+* `GetLogicalVolumeFSTypeCommand`      (string), command which return logical volume filesystem type
 * `CreateSnapshotCommand`              (string), command which creates new LVM snapshot of MySQL data
 * `AvailableLocalSnapshotHostsCommand` (string), command which returns list of hosts in local DC on which recent snapshots are available
 * `AvailableSnapshotHostsCommand`      (string), command which returns list of hosts in all DCs on which recent snapshots are available
@@ -111,6 +118,7 @@ The following is a complete list of configuration parameters:
 * `PostCopyCommand`                    (string), command to be executed after the seed is complete (cleanup)
 * `AgentsServer`                       (string), **Required** URL of your **orchestrator** daemon, You must add the port the orchestrator server expects to talk to agents to (see below, e.g. `https://my.orchestrator.daemon:3001`)
 * `HTTPPort`                           (uint),   Port to listen on  
+* `SeedTransferPort`                   (uint),   TCP Port to seed data transfer  
 * `HTTPAuthUser`                       (string), Basic auth user (default empty, meaning no auth)
 * `HTTPAuthPassword`                   (string), Basic auth password
 * `UseSSL`                             (bool),   If `true` then serving via `https` protocol
@@ -127,6 +135,8 @@ An example configuration file may be:
     "AgentsServer": "https://my.orchestrator.daemon:3001",
     "ContinuousPollSeconds" : 60,
     "ResubmitAgentIntervalMinutes": 60,
+    "LogicalVolumesCommand": "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent",
+    "GetMountCommand":       "grep %s /etc/mtab", 
     "CreateSnapshotCommand":                "/path/to/snapshot-command.bash",
     "AvailableLocalSnapshotHostsCommand":   "/path/to/snapshot-local-availability-command.bash",
     "AvailableSnapshotHostsCommand":        "/path/to/snapshot-availability-command.bash",

conf/orchestrator-agent.conf.json

@@ -4,6 +4,8 @@
     "AgentsServerPort": ":3001",
     "ContinuousPollSeconds" : 60,
     "ResubmitAgentIntervalMinutes": 60,
+    "LogicalVolumesCommand": "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent",
+    "GetMountCommand": "grep %s /etc/mtab",
     "CreateSnapshotCommand": "echo 'no action'",
     "AvailableLocalSnapshotHostsCommand": "echo 127.0.0.1",
     "AvailableSnapshotHostsCommand": "echo localhost\n127.0.0.1",

docker/entrypoint.sh

@@ -2,36 +2,43 @@
 if [ ! -e /etc/orchestrator-agent.conf.json ] ; then
 cat <<EOF > /etc/orchestrator-agent.conf.json
 {
-    "SnapshotMountPoint": "/tmp",
-    "AgentsServer": "http://localhost",
-    "AgentsServerPort": ":3001",
-    "ContinuousPollSeconds" : 60,
-    "ResubmitAgentIntervalMinutes": 60,
-    "CreateSnapshotCommand": "echo 'no action'",
-    "AvailableLocalSnapshotHostsCommand": "echo 127.0.0.1",
-    "AvailableSnapshotHostsCommand": "echo localhost\n127.0.0.1",
-    "SnapshotVolumesFilter": "-my-snapshot-",
-    "MySQLDatadirCommand": "echo '~/tmp'",
-    "MySQLPortCommand": "echo '3306'",
-    "MySQLDeleteDatadirContentCommand": "echo 'will not do'",
-    "MySQLServiceStopCommand":      "/etc/init.d/mysqld stop",
-    "MySQLServiceStartCommand":     "/etc/init.d/mysqld start",
-    "MySQLServiceStatusCommand":    "/etc/init.d/mysqld status",
-    "ReceiveSeedDataCommand":       "echo 'not implemented here'",
-    "SendSeedDataCommand":          "echo 'not implemented here'",
-    "PostCopyCommand":              "echo 'post copy'",
-    "HTTPPort": 3002,
-    "HTTPAuthUser": "",
-    "HTTPAuthPassword": "",
-    "UseSSL": false,
-    "SSLCertFile": "",
-    "SSLPrivateKeyFile": "",
-    "HttpTimeoutSeconds": 10,
-    "ExecWithSudo": false,
+    "SnapshotMountPoint": "${SNAPSHOT_MOUNT_POINT:-/mysql-data}",
+    "AgentsServer": "${AGENTS_SERVER:-http://localhost}",
+    "AgentsServerPort": "${AGENTS_SERVER_PORT:-:3001}",
+    "ContinuousPollSeconds" : ${CONTINUOUS_POLL_SECONDS:-60},
+    "ResubmitAgentIntervalMinutes": ${RESUBMIT_AGENTINTERVAL_MINUTES:-10},
+    "LogicalVolumesCommand": "${LOGICAL_VOLUMES_COMMAND:-lvs --noheading -o lv_name,vg_name,lv_path,snap_percent}",
+    "GetMountCommand":       "${GET_MOUNT_COMMAND:-grep %s /etc/mtab}",
+    "UnmountCommand":        "${UNMOUNT_COMMAND:-umount %s}",
+    "MountLVCommand":        "${MOUNT_LV_COMMAND:-mount %s %s %s}",
+    "RemoveLVCommand":       "${REMOVE_LV_COMMAND:-lvremove --force %s}",
+    "MySQLTailErrorLogCommand": "${MYSQL_TAIL_ERROR_LOG_COMMAND:-tail -n 20 \$(mysql -B --skip-column-names -e \"SELECT @@log_error\")}",
+    "GetLogicalVolumeFSTypeCommand": "${GET_LOGICAL_VOLUME_FS_TYPE_COMMAND:-blkid %s}",
+    "CreateSnapshotCommand": "${CREATE_SNAPSHOT_COMMAND:-echo \'no action\'}",
+    "AvailableLocalSnapshotHostsCommand": "${AVAILABLE_LOCAL_SNAPSHOT_HOSTS_COMMAND:-echo 127.0.0.1}",
+    "AvailableSnapshotHostsCommand": "${AVAILABLE_SNAPSHOT_HOSTS_COMMAND:-printf \'localhost\n127.0.0.1\'}",
+    "SnapshotVolumesFilter": "${SNAPSHOT_VOLUMES_FILTER:--mysql-snapshot-}",
+    "MySQLDatadirCommand": "${MYSQL_DATADIR_COMMAND:-mysql -B --skip-column-names -e 'SELECT @@datadir'}",
+    "MySQLPortCommand": "${MYSQL_PORT_COMMAND:-echo '3306'}",
+    "MySQLDeleteDatadirContentCommand": "${MYSQL_DELETE_DATADIR_CONTENT_COMMAND:-echo 'will not do'}",
+    "MySQLServiceStopCommand":      "${MYSQL_SERVICE_STOP_COMMAND:-/etc/init.d/mysqld stop}",
+    "MySQLServiceStartCommand":     "${MYSQL_SERVICE_START_COMMAND:-/etc/init.d/mysqld start}",
+    "MySQLServiceStatusCommand":    "${MYSQL_SERVICE_STATUS_COMMAND:-/etc/init.d/mysqld status}",
+    "ReceiveSeedDataCommand":       "${RECEIVE_SEED_DATA_COMMAND:-echo \'not implemented here\'}",
+    "SendSeedDataCommand":          "${SEND_SEED_DATA_COMMAND:-echo \'not implemented here\'}",
+    "PostCopyCommand":              "${POST_COPY_COMMAND:-echo \'post copy\'}",
+    "HTTPPort": ${HTTP_PORT:-3002},
+    "HTTPAuthUser": "${HTTP_AUTH_USER}",
+    "HTTPAuthPassword": "${HTTP_AUTH_PASSWORD}",
+    "UseSSL": ${USE_SSL:-false},
+    "SSLCertFile": "${SSL_CERT_FILE}",
+    "SSLPrivateKeyFile": "${SSL_PRIVATE_KEY_FILE}",
+    "HttpTimeoutSeconds": ${HTTP_TIMEOUT_SECONDS:-10},
+    "ExecWithSudo": ${EXEC_WITH_SUDO:-false},
     "CustomCommands": {
         "true": "/bin/true"
     },
-    "TokenHintFile": ""
+    "TokenHintFile": "${TOKEN_HINT_FILE}"
 }
 EOF
 fi

etc/init.d/orchestrator-agent.bash

@@ -26,6 +26,12 @@ DESC="orchestrator-agent: MySQL management agent"
 PIDFILE=/var/run/$NAME.pid
 SCRIPTNAME=/etc/init.d/$NAME
 
+# This files can be used to inject pre-service execution
+# scripts, such as exporting variables or whatever. It's yours!
+[ -f /etc/default/orchestrator-agent ] && . /etc/default/orchestrator-agent
+[ -f /etc/orchestrator-agent_profile ] && . /etc/orchestrator-agent_profile
+[ -f /etc/profile.d/orchestrator-agent ] && . /etc/profile.d/orchestrator-agent
+
 case "$1" in
   start)
     printf "%-50s" "Starting $NAME..."
@@ -60,21 +66,39 @@ case "$1" in
     PID=$(cat $PIDFILE)
     cd $DAEMON_PATH
     if [ -f $PIDFILE ]; then
-      kill -HUP $PID
-      printf "%s\n" "Ok"
+      kill -TERM $PID
       rm -f $PIDFILE
+      # Wait for orchestrator-agent to stop otherwise restart may fail.
+      # (The newly restarted process may be unable to bind to the
+      # currently bound socket.)
+      while ps -p $PID >/dev/null 2>&1; do
+        printf "."
+        sleep 1
+      done
+      printf "\n"
+      printf "Ok\n"
     else
       printf "%s\n" "pidfile not found"
       exit 1
     fi
   ;;
-
   restart)
     $0 stop
     $0 start
   ;;
-
+  reload)
+    printf "%-50s" "Reloading $NAME"
+    PID=$(cat $PIDFILE)
+    cd $DAEMON_PATH
+    if [ -f $PIDFILE ]; then
+      kill -HUP $PID
+      printf "%s\n" "Ok"
+    else
+      printf "%s\n" "pidfile not found"
+      exit 1
+    fi
+	;;
   *)
-    echo "Usage: $0 {status|start|stop|restart}"
+    echo "Usage: $0 {status|start|stop|restart|reload}"
     exit 1
 esac

go/cmd/orchestrator-agent/main.go

@@ -33,11 +33,20 @@ var AppVersion string
 
 func acceptSignal() {
 	c := make(chan os.Signal, 1)
-	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGHUP)
 
 	// Block until a signal is received.
-	sig := <-c
-	log.Fatalf("Got signal: %+v", sig)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGTERM)
+	for sig := range c {
+		switch sig {
+		case syscall.SIGHUP:
+			log.Infof("Received SIGHUP. Reloading configuration")
+			config.Reload()
+		case syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT:
+			log.Infof("Received %s. Shutting down orchestrator-agent", sig.String())
+			// probably should poke other go routines to stop cleanly here ...
+			os.Exit(0)
+		}
+	}
 }
 
 // main is the application's entry point. It will either spawn a CLI or HTTP itnerfaces.

go/config/config.go

@@ -28,6 +28,13 @@ type Configuration struct {
 	SnapshotMountPoint                 string            // The single, agreed-upon mountpoint for logical volume snapshots
 	ContinuousPollSeconds              uint              // Poll interval for continuous operation
 	ResubmitAgentIntervalMinutes       uint              // Poll interval for resubmitting this agent on orchestrator agents API
+	LogicalVolumesCommand              string            // Command which list logical volumes, default implementation used lvs
+	GetMountCommand                    string            // Command which get mount point parameters by mount point name, default implementation over cat %s/etc/mtab
+	UnmountCommand                     string            // Command which unmount current mount point, default implementation just execute `umount %s`
+	MountLVCommand                     string            // Command which mount selected snapshot into mount point
+	RemoveLVCommand                    string            // Command which remove selected snapshot from disk
+	MySQLTailErrorLogCommand           string            // Command which return last 20 lines from @@log_error file
+	GetLogicalVolumeFSTypeCommand      string            // Command which return logical volume filesystem type
 	CreateSnapshotCommand              string            // Command which creates a snapshot logical volume. It's a "do it yourself" implementation
 	AvailableLocalSnapshotHostsCommand string            // Command which returns list of hosts (one host per line) with available snapshots in local datacenter
 	AvailableSnapshotHostsCommand      string            // Command which returns list of hosts (one host per line) with available snapshots in any datacenter
@@ -45,6 +52,7 @@ type Configuration struct {
 	AgentsServer                       string            // HTTP address of the orchestrator agents server
 	AgentsServerPort                   string            // HTTP port of the orchestrator agents server
 	HTTPPort                           uint              // HTTP port on which this service listens
+	SeedTransferPort                   uint              // TCP port for data seed transfer
 	HTTPAuthUser                       string            // Username for HTTP Basic authentication (blank disables authentication)
 	HTTPAuthPassword                   string            // Password for HTTP Basic authentication
 	UseSSL                             bool              // If true, service will serve HTTPS only
@@ -65,13 +73,21 @@ type Configuration struct {
 }
 
 var Config = NewConfiguration()
+var configFileNames []string
 
 func NewConfiguration() *Configuration {
 	return &Configuration{
 		SnapshotMountPoint:                 "",
 		ContinuousPollSeconds:              60,
 		ResubmitAgentIntervalMinutes:       60,
 		CreateSnapshotCommand:              "",
+		LogicalVolumesCommand:              "lvs --noheading -o lv_name,vg_name,lv_path,snap_percent",
+		GetMountCommand:                    "grep %s /etc/mtab",
+		UnmountCommand:                     "umount %s",
+		MountLVCommand:                     "mount %s %s %s",
+		RemoveLVCommand:                    "lvremove --force %s",
+		MySQLTailErrorLogCommand:           `tail -n 20 $(egrep "log[-_]error" /etc/my.cnf | cut -d "=" -f 2)`,
+		GetLogicalVolumeFSTypeCommand:      "blkid %s",
 		AvailableLocalSnapshotHostsCommand: "",
 		AvailableSnapshotHostsCommand:      "",
 		SnapshotVolumesFilter:              "",
@@ -88,6 +104,7 @@ func NewConfiguration() *Configuration {
 		AgentsServer:                       "",
 		AgentsServerPort:                   "",
 		HTTPPort:                           3002,
+		SeedTransferPort:                   21234,
 		HTTPAuthUser:                       "",
 		HTTPAuthPassword:                   "",
 		UseSSL:                             false,
@@ -130,9 +147,15 @@ func Read(file_names ...string) *Configuration {
 	for _, file_name := range file_names {
 		read(file_name)
 	}
+	configFileNames = file_names
 	return Config
 }
 
+// Reload
+func Reload() *Configuration {
+	return Read(configFileNames...)
+}
+
 // ForceRead reads configuration from given file name or bails out if it fails
 func ForceRead(file_name string) *Configuration {
 	_, err := read(file_name)

go/http/api.go

@@ -363,7 +363,11 @@ func (this *HttpAPI) PostCopy(params martini.Params, r render.Render, req *http.
 	if err := this.validateToken(r, req); err != nil {
 		return
 	}
-	err := osagent.PostCopy()
+	qs := req.URL.Query()
+	if sourceHost, exists := params["sourceHost"]; !exists || sourceHost=="" {
+		params["sourceHost"] = qs.Get("sourceHost")
+	}
+	err := osagent.PostCopy(params["sourceHost"])
 	if err != nil {
 		r.JSON(500, &APIResponse{Code: ERROR, Message: err.Error()})
 		return
@@ -617,6 +621,7 @@ func (this *HttpAPI) RegisterRequests(m *martini.ClassicMartini) {
 	m.Get("/api/delete-mysql-datadir", this.DeleteMySQLDataDir)
 	m.Get("/api/mysql-datadir-available-space", this.GetMySQLDataDirAvailableDiskSpace)
 	m.Get("/api/post-copy", this.PostCopy)
+	m.Get("/api/post-copy/:sourceHost", this.PostCopy)
 	m.Get("/api/receive-mysql-seed-data/:seedId", this.ReceiveMySQLSeedData)
 	m.Get("/api/send-mysql-seed-data/:targetHost/:seedId", this.SendMySQLSeedData)
 	m.Get("/api/abort-seed/:seedId", this.AbortSeed)