Keygen automatic restart possibility

demo/validator1/.env.development

@@ -9,7 +9,7 @@ SIDE_SHARED_DB_ADDRESS=0xd5fE0D28e058D375b0b038fFbB446Da37E85fFdc
 
 FOREIGN_URL=http://http-api:8000
 FOREIGN_CHAIN_ID=Binance-Dev
-FOREIGN_ASSET=DEV-BA8
+FOREIGN_ASSET=DEV-9BA
 FOREIGN_FETCH_MAX_TIME_INTERVAL=60000
 FOREIGN_FETCH_INTERVAL=5000
 FOREIGN_FETCH_BLOCK_TIME_OFFSET=10000
@@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000
 
+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 VALIDATOR_PRIVATE_KEY=2be3f252e16541bf1bb2d4a517d2bf173e6d09f2d765d32c64dc50515aec63ea
 
 VOTES_PROXY_PORT=5001

demo/validator1/.env.staging

@@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000
 
+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 #VALIDATOR_PRIVATE_KEY is taken from .keys
 
 VOTES_PROXY_PORT=5001

demo/validator2/.env.development

@@ -9,7 +9,7 @@ SIDE_SHARED_DB_ADDRESS=0xd5fE0D28e058D375b0b038fFbB446Da37E85fFdc
 
 FOREIGN_URL=http://http-api:8000
 FOREIGN_CHAIN_ID=Binance-Dev
-FOREIGN_ASSET=DEV-BA8
+FOREIGN_ASSET=DEV-9BA
 FOREIGN_FETCH_MAX_TIME_INTERVAL=60000
 FOREIGN_FETCH_INTERVAL=5000
 FOREIGN_FETCH_BLOCK_TIME_OFFSET=10000
@@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000
 
+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 VALIDATOR_PRIVATE_KEY=e59d58c77b791f98f10187117374ae9c589d48a62720ec6a5e142b0cc134f685
 
 VOTES_PROXY_PORT=5002

demo/validator2/.env.staging

@@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000
 
+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 #VALIDATOR_PRIVATE_KEY is taken from .keys
 
 VOTES_PROXY_PORT=5002

demo/validator3/.env.development

@@ -9,7 +9,7 @@ SIDE_SHARED_DB_ADDRESS=0xd5fE0D28e058D375b0b038fFbB446Da37E85fFdc
 
 FOREIGN_URL=http://http-api:8000
 FOREIGN_CHAIN_ID=Binance-Dev
-FOREIGN_ASSET=DEV-BA8
+FOREIGN_ASSET=DEV-9BA
 FOREIGN_FETCH_MAX_TIME_INTERVAL=60000
 FOREIGN_FETCH_INTERVAL=5000
 FOREIGN_FETCH_BLOCK_TIME_OFFSET=10000
@@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000
 
+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 VALIDATOR_PRIVATE_KEY=afaa4d4d6e54d25b0bf0361e3fd6cef562f6311bf6200de2dd0aa4cab63ae3b5
 
 VOTES_PROXY_PORT=5003

demo/validator3/.env.staging

@@ -18,6 +18,9 @@ SIGN_ATTEMPT_TIMEOUT=120000
 SIGN_NONCE_CHECK_INTERVAL=10000
 SEND_TIMEOUT=60000
 
+KEYGEN_ATTEMPT_TIMEOUT=120000
+KEYGEN_EPOCH_CHECK_INTERVAL=10000
+
 #VALIDATOR_PRIVATE_KEY is taken from .keys
 
 VOTES_PROXY_PORT=5003

src/oracle/docker-compose-test.yml

@@ -34,6 +34,8 @@ services:
     environment:
       - 'RABBITMQ_URL=amqp://rabbitmq:5672'
       - 'PROXY_URL=http://local_proxy:8001'
+      - KEYGEN_ATTEMPT_TIMEOUT
+      - KEYGEN_EPOCH_CHECK_INTERVAL
       - LOG_LEVEL
     volumes:
       - '${PWD}/${TARGET_NETWORK}/keys:/keys'

src/oracle/docker-compose.yml

@@ -31,6 +31,8 @@ services:
     environment:
       - 'RABBITMQ_URL=amqp://rabbitmq:5672'
       - 'PROXY_URL=http://proxy:8001'
+      - KEYGEN_ATTEMPT_TIMEOUT
+      - KEYGEN_EPOCH_CHECK_INTERVAL
       - LOG_LEVEL
     volumes:
       - '${PWD}/${TARGET_NETWORK}/keys:/keys'

src/oracle/proxy/index.js

@@ -73,6 +73,19 @@ function sideSendQuery(query) {
   })
 }
 
+async function status(req, res) {
+  logger.debug('Status call')
+  const [bridgeEpoch, bridgeStatus] = await Promise.all([
+    bridge.epoch(),
+    bridge.status()
+  ])
+  res.send({
+    bridgeEpoch,
+    bridgeStatus
+  })
+  logger.debug('Status end')
+}
+
 async function get(req, res) {
   logger.debug('Get call, %o', req.body.key)
   const round = req.body.key.second
@@ -131,12 +144,26 @@ async function signupKeygen(req, res) {
   const epoch = await bridge.nextEpoch()
   const partyId = await bridge.getNextPartyId(validatorAddress)
 
+  logger.debug('Checking previous attempts')
+  let attempt = 1
+  let uuid
+  while (true) {
+    uuid = `k${epoch}_${attempt}`
+    const data = await sharedDb.getData(validatorAddress, ethers.utils.id(uuid), ethers.utils.id('round1_0'))
+    if (data.length === 2) {
+      break
+    }
+    logger.trace(`Attempt ${attempt} is already used`)
+    attempt += 1
+  }
+  logger.debug(`Using attempt ${attempt}`)
+
   if (partyId === 0) {
     res.send(Err({ message: 'Not a validator' }))
     logger.debug('Not a validator')
   } else {
     res.send(Ok({
-      uuid: `k${epoch}`,
+      uuid,
       number: partyId
     }))
     logger.debug('SignupKeygen end')
@@ -287,6 +314,22 @@ async function voteChangeThreshold(req, res) {
   }
 }
 
+async function voteChangeRangeSize(req, res) {
+  if (/^[0-9]+$/.test(req.params.rangeSize)) {
+    logger.info('Voting for changing range size')
+    const epoch = await bridge.epoch()
+    const message = buildMessage(
+      Action.VOTE_CHANGE_RANGE_SIZE,
+      epoch,
+      parseInt(req.params.rangeSize, 10),
+      padZeros(req.attempt, 54)
+    )
+    await processMessage(message)
+    res.send('Voted\n')
+    logger.info('Voted successfully')
+  }
+}
+
 async function voteChangeCloseEpoch(req, res) {
   if (req.params.closeEpoch === 'true' || req.params.closeEpoch === 'false') {
     logger.info('Voting for changing close epoch')
@@ -350,7 +393,7 @@ async function info(req, res) {
   try {
     const [
       x, y, epoch, rangeSize, nextRangeSize, closeEpoch, nextCloseEpoch, epochStartBlock,
-      foreignNonce, nextEpoch, threshold, nextThreshold, validators, nextValidators, status,
+      foreignNonce, nextEpoch, threshold, nextThreshold, validators, nextValidators, bridgeStatus,
       homeBalance
     ] = await Promise.all([
       bridge.getX().then((value) => new BN(value).toString(16)),
@@ -394,7 +437,7 @@ async function info(req, res) {
       homeBalance,
       foreignBalanceTokens: parseFloat(balances[FOREIGN_ASSET]) || 0,
       foreignBalanceNative: parseFloat(balances.BNB) || 0,
-      bridgeStatus: decodeStatus(status)
+      bridgeStatus: decodeStatus(bridgeStatus)
     }
     logger.trace('%o', msg)
     res.send(msg)
@@ -408,6 +451,8 @@ async function info(req, res) {
   logger.debug('Info end')
 }
 
+app.get('/status', status)
+
 app.post('/get', get)
 app.post('/set', set)
 app.post('/signupkeygen', signupKeygen)
@@ -437,6 +482,7 @@ votesProxyApp.use('/vote', (req, res, next) => {
 votesProxyApp.get('/vote/addValidator/:validator', voteAddValidator)
 votesProxyApp.get('/vote/removeValidator/:validator', voteRemoveValidator)
 votesProxyApp.get('/vote/changeThreshold/:threshold', voteChangeThreshold)
+votesProxyApp.get('/vote/changeRangeSize/:rangeSize', voteChangeRangeSize)
 votesProxyApp.get('/vote/changeCloseEpoch/:closeEpoch', voteChangeCloseEpoch)
 votesProxyApp.get('/info', info)
 

src/oracle/tss-keygen/keygen.js

@@ -9,6 +9,12 @@ const { publicKeyToAddress } = require('./crypto')
 const { delay } = require('./wait')
 
 const { RABBITMQ_URL, PROXY_URL } = process.env
+const KEYGEN_ATTEMPT_TIMEOUT = parseInt(process.env.KEYGEN_ATTEMPT_TIMEOUT, 10)
+const KEYGEN_EPOCH_CHECK_INTERVAL = parseInt(process.env.KEYGEN_EPOCH_CHECK_INTERVAL, 10)
+
+const KEYGEN_OK = 0
+const KEYGEN_EPOCH_INTERRUPT = 1
+const KEYGEN_FAILED = 2
 
 const app = express()
 
@@ -34,6 +40,54 @@ function writeParams(parties, threshold) {
   }))
 }
 
+function killKeygen() {
+  exec.execSync('pkill gg18_keygen || true')
+}
+
+function keygen(keysFile, epoch) {
+  let restartTimeoutId
+  let epochDaemonIntervalId
+  let epochInterrupt
+
+  return new Promise((resolve) => {
+    const cmd = exec.execFile('./keygen-entrypoint.sh', [PROXY_URL, keysFile], (error) => {
+      logger.trace('Keygen entrypoint exited, %o', error)
+      clearTimeout(restartTimeoutId)
+      clearInterval(epochDaemonIntervalId)
+      currentKeygenEpoch = null
+      if (fs.existsSync(keysFile)) {
+        logger.info(`Finished keygen for epoch ${epoch}`)
+        resolve(KEYGEN_OK)
+      } else {
+        logger.warn(`Keygen for epoch ${epoch} failed, will start new attempt`)
+        resolve(epochInterrupt ? KEYGEN_EPOCH_INTERRUPT : KEYGEN_FAILED)
+      }
+    })
+    cmd.stdout.on('data', (data) => {
+      const str = data.toString()
+      if (str.includes('Got all party signups')) {
+        restartTimeoutId = setTimeout(killKeygen, KEYGEN_ATTEMPT_TIMEOUT)
+      }
+      logger.debug(str)
+    })
+    cmd.stderr.on('data', (data) => logger.debug(data.toString()))
+
+    // Kill keygen if keygen for current epoch is already confirmed
+    epochDaemonIntervalId = setInterval(async () => {
+      logger.info(`Checking if bridge has confirmations keygen for epoch ${epoch}`)
+      const { bridgeEpoch, bridgeStatus } = (await proxyClient.get('/status')).data
+      logger.trace(`Current bridge epoch: ${bridgeEpoch}, current bridge status: ${bridgeStatus}`)
+      if (bridgeEpoch > epoch || bridgeStatus > 3) {
+        logger.info(`Bridge has already confirmed keygen for epoch ${epoch}`)
+        epochInterrupt = true
+        // Additional delay, maybe keygen will eventually finish
+        await delay(5000)
+        killKeygen()
+      }
+    }, KEYGEN_EPOCH_CHECK_INTERVAL)
+  })
+}
+
 async function keygenConsumer(msg) {
   const { epoch, parties, threshold } = JSON.parse(msg.content)
   logger.info(`Consumed new epoch event, starting keygen for epoch ${epoch}`)
@@ -44,23 +98,26 @@ async function keygenConsumer(msg) {
   currentKeygenEpoch = epoch
 
   writeParams(parties, threshold)
-  const cmd = exec.execFile('./keygen-entrypoint.sh', [PROXY_URL, keysFile], async () => {
-    currentKeygenEpoch = null
-    if (fs.existsSync(keysFile)) {
-      logger.info(`Finished keygen for epoch ${epoch}`)
+
+  while (true) {
+    const keygenResult = await keygen(keysFile, epoch)
+
+    if (keygenResult === KEYGEN_OK) {
       const publicKey = JSON.parse(fs.readFileSync(keysFile))[5]
       logger.warn(`Generated multisig account in binance chain: ${publicKeyToAddress(publicKey)}`)
 
       logger.info('Sending keys confirmation')
       await confirmKeygen(publicKey, epoch)
-    } else {
-      logger.warn(`Keygen for epoch ${epoch} failed`)
+      break
+    } else if (keygenResult === KEYGEN_EPOCH_INTERRUPT) {
+      logger.warn('Keygen was interrupted by epoch daemon')
+      break
     }
-    logger.debug('Ack for keygen message')
-    channel.ack(msg)
-  })
-  cmd.stdout.on('data', (data) => logger.debug(data.toString()))
-  cmd.stderr.on('data', (data) => logger.debug(data.toString()))
+
+    await delay(1000)
+  }
+  logger.info('Acking message')
+  channel.ack(msg)
 }
 
 async function main() {
@@ -81,7 +138,7 @@ async function main() {
     logger.info(`Consumed new cancel event for epoch ${epoch} keygen`)
     if (currentKeygenEpoch === epoch) {
       logger.info('Cancelling current keygen')
-      exec.execSync('pkill gg18_keygen || true')
+      killKeygen()
     }
     channel.ack(msg)
   })

src/oracle/tss-sign/signer.js

@@ -213,6 +213,8 @@ function sign(keysFile, tx, publicKey, signerAddress) {
       if (sequence > tx.tx.sequence) {
         logger.info('Account already has needed nonce, cancelling current sign process')
         nonceInterrupt = true
+        // Additional delay, maybe signer will eventually finish
+        await delay(5000)
         killSigner()
       }
     }, SIGN_NONCE_CHECK_INTERVAL)