Intermittent error "This socket has been ended by the other party"

[vpsouza]

Hi, i'm connecting to my neo4j running on a docker container and i'm getting an error when i try to attempt to make another connection/query. The error is "This socket has been ended by the other party". This is my Dockerfile with the opened ports:

EXPOSE 7474 7473 7687

This is my driver utility:

var neo4j = require("neo4j-driver").v1,
    config = require("../../config");

(function(module){ 
    module.exports = function(){
        config.neo4j.server = 'localhost';
        return neo4j.driver("bolt://" + config.neo4j.server, neo4j.auth.basic(config.neo4j.user, config.neo4j.password));
    }
}(module));

The way i'm making my queries to neo4j:

(function(module){
    var driver = require('../../../utils/neo4j-driver')(),
        Q = require('q'),
        logger = require('../../../utils/logger'),
        BaseNeo4jModel = require('../../../utils/neo4j-model');

    function userDBAuth(user){
        var deferred = Q.defer();
        var session = driver.session();

        session
            //.run( "MATCH (a:Person) RETURN a.name AS name, a.uuid as uuid" )
            .run("MATCH (user:User {email:{email}})"
                    + "RETURN user",
                    { email: user.email})
            .then(function (result) {
                logger.debug('fetching results...');

                if(result.records.length > 0){
                    var records = [];

                    result.records.forEach(function(record){
                        records.push(new BaseNeo4jModel(record));
                    });

                    done();

                    deferred.resolve(records);
                } else {
                    deferred.reject({sucess: false, message: 'User not Found'});
                }

            }).catch( function(err) {
                logger.error(err);
                done();
                deferred.reject(err.fields[0]);
            });

            return deferred.promise;

        function done(){
            session.close();
            driver.close();
        }
    }

    module.exports = userDBAuth;

}(module));

And this is my stack trace:

{"name":"auth-services","hostname":"MacBook-Pro-de-Vinicius.local","pid":16292,"level":50,"err":{"message":"This socket has been ended by the other party","name":"Error","stack":"Error: This socket has been ended by the other party
at TLSSocket.writeAfterFIN [as write] (net.js:286:12)
at NodeChannel.write (/Users/viniciussouza/Sites/WeddSocial/backend/weddsocial-auth/node_modules/neo4j-driver/lib/v1/internal/ch-node.js:285:20)
at Chunker.flush (/Users/viniciussouza/Sites/WeddSocial/backend/weddsocial-auth/node_modules/neo4j-driver/lib/v1/internal/chunking.js:114:18)
at Connection.sync (/Users/viniciussouza/Sites/WeddSocial/backend/weddsocial-auth/node_modules/neo4j-driver/lib/v1/internal/connector.js:487:21)
at Session.run (/Users/viniciussouza/Sites/WeddSocial/backend/weddsocial-auth/node_modules/neo4j-driver/lib/v1/session.js:89:20)
at userDBAuth (/Users/viniciussouza/Sites/WeddSocial/backend/weddsocial-auth/app/api/auth/services/userDBAuth.js:13:7)
at Object._auth [as auth] (/Users/viniciussouza/Sites/WeddSocial/backend/weddsocial-auth/app/api/auth/services/auth.js:9:3)
at Server.auth (/Users/viniciussouza/Sites/WeddSocial/backend/weddsocial-auth/app/api/auth/resources/auth.js:7:12)
at next (/Users/viniciussouza/Sites/WeddSocial/backend/weddsocial-auth/node_modules/restify/lib/server.js:906:30)
at f (/Users/viniciussouza/Sites/WeddSocial/backend/weddsocial-auth/node_modules/once/once.js:25:25)","code":"EPIPE"},"msg":"This socket has been ended by the other party","time":"2016-09-27T11:12:33.163Z","v":0}

I'm desperate, please help me!

[vpsouza]

Same issue, with other bolt driver (this time java driver)....

Neo4j Bolt driver #229

[viz]

just saw the same symptoms with the javascript bolt driver. Using version 1.1.0-M02 : Query failing with this error:

at TLSSocket.writeAfterFIN [as write] (net.js line 284 col 12)
2
at NodeChannel.write (/app/node_modules/neo4j-driver/lib/v1/internal/ch-node.js line 329 col 20)
this._conn.write(buffer._buffer);
3
at Chunker.flush (/app/node_modules/neo4j-driver/lib/v1/internal/chunking.js line 114 col 18)
this._ch.write(out.getSlice(0, out.position));
4
at Connection.sync (/app/node_modules/neo4j-driver/lib/v1/internal/connector.js line 489 col 21)
this._chunker.flush();
5
at Session.run (/app/node_modules/neo4j-driver/lib/v1/session.js line 102 col 20)
this._conn.sync();
6
at Order.updateAllRights (/app/models/order.js line 300 col 13)
dbSession.run(query, queryParams)

[viz]

is anyone maintaining this driver? There's been no post on this issue for more than a month..

[dozoisch]

@viz I had the same issue, I checked the latest version of the driver from GitHub and so far so good. I'll update you if it starts again. "neo4j-driver": "neo4j/neo4j-javascript-driver#56c7995cd26bd8fc9b7769b2a42e99bb24087c80",

[dozoisch]

No, nevermind, still happening.

[zhenlineo]

Hi, sorry for long waiting time.
Some question to help to locate the error: Do you have any load balancer in the middle? Do you have any timeout on the connection with the bolt driver?

Could @vpsouza you share the docker image that you are using and give us some instruction how to re-produce this error?

[dozoisch]

@zhenlineo I'm using HAProxy in the middle, configured as it was said in the HAProxy doc something really close to this

This is the stacktrace based on commit 56c7995

Error: This socket has been ended by the other party
  at TLSSocket.writeAfterFIN [as write] (net.js, line 291 col 12)
  at NodeChannel.write (node_modules/neo4j-driver/lib/v1/internal/ch-node.js, line 329 col 20) 
  at Chunker.flush (node_modules/neo4j-driver/lib/v1/internal/chunking.js, line 114 col 18)
  at Connection.sync (node_modules/neo4j-driver/lib/v1/internal/connector.js, line 489 col 21)
  at Session.run (node_modules/neo4j-driver/lib/v1/session.js, line 102 col 20)

[zhenlineo]

Hi @dozoisch, your connection will get killed by HAProxy after 30s timeout? (Not sure the HAProxy setting means an idle timeout or just kill the tcp connection after 30s)

Some more info about this neo4j/neo4j-java-driver#213
Where a user provides a nice suggestion to do re-tries to avoid the problem.

[dozoisch]

@zhenlineo so the "timeout connect" is timeout when establishing the connection. The idle timeout is the "timeout client". (see http://serverfault.com/a/778222/138439 for more info)

I'm currently putting in place a retry mechanism. So that when this particular error happen I retry a number of times. I'll report on how that goes!

[dozoisch]

So I'm doing something similar to this:

const maxRetry = 4;
function doRun(cypher, params, numberOfTry) {
  const session = driver.session();
  return Promise.resolve().then(() => session.run(cypher, params))
    .then(
      results => {
        session.close();
        return results;
      },
      err => {
        session.close();
        if (numberOfTry < maxRetry && err.message.indexOf('ended by the other party') > -1) {
          // Try again
          return doRun(cypher, params, ++numberOfTry);
        }
        return Promise.reject(err);
      }
    );
}

exports.run = function(cypher, params) {
  return doRun(cypher, params, 0);
};  

And it seems to be working so far. I've put the retry count higher than what is probably needed, but I'm logging when it happens so I'll be able to tweak it. It seems to be working so far!

It's not the perfect solution, but it does work around the issue.

[zhenlineo]

@dozoisch You might want to set the retry time to be higher than pool size which you could set in the config when you construct the driver. The driver will first try to reuse connections from the connection pool and then roll back to create new connections. So for the worst case, you need to retry poolSize + 1 times to get a fresh connection with the server.

[dozoisch]

Oh good idea!

[louisiukas]

I get this error in the following scenario:

• Using neo4j 3.0.7 docker image and neo4j-javascript-driver 1.0.5
• I'm setting up the connection to the db once using const neo4j=require('neo4j-driver').v1; const neo4jDriver =neo4j.driver(..), while using short lived sessions.
• If the database has been down, but is up again and ready, at least the next session I request (var session=neo4jDriver.session();) will fail.
  • It fails once it is used: session.run(...);
• This is a bug in my book. A fresh session should not fail if the db is ready.
• After one or more failed sessions, the connection works again.

[lutovich]

Hello @vpsouza, @viz, @louisiukas. Do you also use any proxy or load balancer between JS driver and Neo4j database?

Current implementation of the connection pool does not have any "test on borrow" or "idle connection refresh" functionality. So if idle pooled connection is killed/broken it'll just be given to the user code as is.