fix(broker/mysql_connection): mysql_ping must be called just when needed

centreon-broker/core/inc/com/centreon/broker/mysql_connection.hh

@@ -73,6 +73,7 @@ class mysql_connection {
   std::list<std::unique_ptr<database::mysql_task>> _tasks_list;
   std::atomic_int _tasks_count;
   bool _need_commit;
+  std::time_t _last_access;
 
   std::unordered_map<uint32_t, MYSQL_STMT*> _stmt;
   std::unordered_map<uint32_t, std::string> _stmt_query;

centreon-broker/core/src/mysql_connection.cc

@@ -131,6 +131,7 @@ bool mysql_connection::_try_to_reconnect() {
         "mysql_connection: The mysql/mariadb database seems not started.");
     return false;
   }
+  _last_access = std::time(nullptr);
 
   _prepare_connection();
 
@@ -224,6 +225,8 @@ void mysql_connection::_commit(mysql_task* t) {
       log_v2::sql()->error("mysql_connection: {}", err_msg);
       std::this_thread::sleep_for(std::chrono::milliseconds(50));
     }
+    if (res == 0)
+      _last_access = std::time(nullptr);
   } else
     res = 0;
 
@@ -586,6 +589,7 @@ void mysql_connection::_run() {
       _start_condition.notify_all();
       return;
     }
+    _last_access = std::time(nullptr);
   }
 
   if (config::applier::mode == config::applier::finished) {
@@ -616,22 +620,25 @@ void mysql_connection::_run() {
                                          static_cast<int>(_tasks_count));
         /* We are waiting for some activity, nothing to do for now it is time
          * to make some ping */
-        while (!_tasks_condition.wait_for(
-            lock, std::chrono::seconds(30),
-            [this] { return _finish_asked || !_tasks_list.empty(); })) {
+        _tasks_condition.wait(
+            lock,
+            [this] { return _finish_asked || !_tasks_list.empty(); });
+
+        std::time_t now = std::time(nullptr);;
+        if (_tasks_list.empty())
+          _state = finished;
+        else if (now >= _last_access + 30) {
           lock.unlock();
           log_v2::sql()->trace("SQL: performing mysql_ping.");
           if (mysql_ping(_conn)) {
             if (!_try_to_reconnect())
               log_v2::sql()->error("SQL: Reconnection failed.");
-          } else
+          } else {
             log_v2::sql()->trace("SQL: connection always alive");
+            _last_access = now;
+          }
           lock.lock();
         }
-
-        if (_tasks_list.empty()) {
-          _state = finished;
-        }
         continue;
       }
 

centreon-tests/broker-database/networkFailure.robot

@@ -1,64 +1,194 @@
 *** Settings ***
-Resource			../resources/resources.robot
-Suite Setup			Clean Before Suite
-Suite Teardown		Clean After Suite
-Test Setup			Stop Processes
+Resource	../resources/resources.robot
+Suite Setup	Clean Before Suite
+Suite Teardown	Clean After Suite
+Test Setup	Stop Processes
 
-Documentation		Centreon Broker database connection failure
-Library				Process
-Library				DateTime
-Library				OperatingSystem
-Library				../resources/BrokerDatabase.py
-Library				../resources/Common.py
-Library				../resources/Broker.py
-Library				../resources/Engine.py
+Documentation	Centreon Broker database connection failure
+Library		Process
+Library		DateTime
+Library		OperatingSystem
+Library		../resources/Common.py
+Library		../resources/Broker.py
+Library		../resources/Engine.py
 
 *** Test Cases ***
 NetworkDbFail1
-	[Documentation]		network failure test between broker and database (shutting down connection for 100ms)
-	[Tags]				Broker		Database	Network
-	Network Failure		interval=100ms
+	[Documentation]	network failure test between broker and database (shutting down connection for 100ms)
+	[Tags]	Broker	Database	Network
+	Network Failure	interval=100ms
+
 NetworkDbFail2
-	[Documentation]		network failure test between broker and database (shutting down connection for 1s)
-	[Tags]				Broker		Database	Network
-	Network Failure		interval=1s
+	[Documentation]	network failure test between broker and database (shutting down connection for 1s)
+	[Tags]	Broker	Database	Network
+	Network Failure	interval=1s
+
 NetworkDbFail3
-	[Documentation]		network failure test between broker and database (shutting down connection for 10s)
-	[Tags]				Broker		Database 	Network
-	Network Failure		interval=10s
+	[Documentation]	network failure test between broker and database (shutting down connection for 10s)
+	[Tags]	Broker	Database	Network
+	Network Failure	interval=10s
+
 NetworkDbFail4
-	[Documentation]		network failure test between broker and database (shutting down connection for 30s)
-	[Tags]				Broker		Database 	Network
-	Network Failure		interval=30s
+	[Documentation]	network failure test between broker and database (shutting down connection for 30s)
+	[Tags]	Broker	Database	Network
+	Network Failure	interval=30s
+
 NetworkDbFail5
-	[Documentation]		network failure test between broker and database (shutting down connection for 60s)
-	[Tags]				Broker		Database 	Network
-	Network Failure		interval=1m
+	[Documentation]	network failure test between broker and database (shutting down connection for 60s)
+	[Tags]	Broker	Database	Network
+	Network Failure	interval=1m
+
+NetworkDBFail6
+	[Documentation]	network failure test between broker and database: we wait for the connection to be established and then we shut down the connection for 60s
+        [Tags]	Broker	Database	Network
+	Config Engine	${1}
+        Config Broker	central
+	Broker Config Output Set	central	central-broker-master-sql	db_host	127.0.0.1
+	Broker Config Output set	central	central-broker-master-sql	connections_count	5
+	Broker Config Output Set	central	central-broker-master-perfdata	db_host	127.0.0.1
+	Broker Config Output set	central	central-broker-master-perfdata	connections_count	5
+	Broker Config Log	central	sql	trace
+        Config Broker	rrd
+        Config Broker	module
+	${start}=	Get Current Date
+	Start Broker
+	Start Engine
+	${result}=	Check Connections
+	Should Be True	${result}	msg=Broker and Engine are not connected
+	${content}=	Create List	run query: SELECT
+	${result}=	Find In Log With Timeout	${logCbd}	${start}	${content}	40
+	Should Be True	${result}	msg=No SELECT done by broker in the DB
+        Disable Eth Connection On Port	port=3306
+        Sleep	1m
+        Reset Eth Connection
+        ${content}=	Create List	0 events acknowledged
+	${result}=	Find In Log With Timeout	${logCbd}	${start}	${content}	40
+        Stop Engine
+        Stop Broker
+
+NetworkDBFailU6
+	[Documentation]	network failure test between broker and database: we wait for the connection to be established and then we shut down the connection for 60s (with unified_sql)
+        [Tags]	Broker	Database	Network	unified_sql
+	Config Engine	${1}
+        Config Broker	central
+	Config Broker Sql Output	central	unified_sql
+	Broker Config Output Set	central	central-broker-unified-sql	db_host	127.0.0.1
+	Broker Config Output set	central	central-broker-master-sql	connections_count	5
+	Broker Config Log	central	sql	trace
+        Config Broker	rrd
+        Config Broker	module
+	${start}=	Get Current Date
+	Start Broker
+	Start Engine
+	${result}=	Check Connections
+	Should Be True	${result}	msg=Broker and Engine are not connected
+	${content}=	Create List	run query: SELECT
+	${result}=	Find In Log With Timeout	${logCbd}	${start}	${content}	40
+	Should Be True	${result}	msg=No SELECT done by broker in the DB
+        Disable Eth Connection On Port	port=3306
+        Sleep	1m
+        Reset Eth Connection
+        ${content}=	Create List	0 events acknowledged
+	${result}=	Find In Log With Timeout	${logCbd}	${start}	${content}	40
+        Stop Engine
+        Stop Broker
+
+NetworkDBFail7
+	[Documentation]	network failure test between broker and database: we wait for the connection to be established and then we shut down the connection for 60s
+        [Tags]	Broker	Database	Network
+	Config Engine	${1}
+        Config Broker	central
+        Reset Eth Connection
+	Broker Config Output Set	central	central-broker-master-sql	db_host	127.0.0.1
+	Broker Config Output set	central	central-broker-master-sql	connections_count	5
+	Broker Config Output Set	central	central-broker-master-perfdata	db_host	127.0.0.1
+	Broker Config Output set	central	central-broker-master-perfdata	connections_count	5
+	Broker Config Log	central	sql	trace
+        Config Broker	rrd
+        Config Broker	module
+	${start}=	Get Current Date
+	Start Broker
+	Start Engine
+	${result}=	Check Connections
+	Should Be True	${result}	msg=Broker and Engine are not connected
+	${content}=	Create List	run query: SELECT
+	${result}=	Find In Log With Timeout	${logCbd}	${start}	${content}	40
+	Should Be True	${result}	msg=No SELECT done by broker in the DB
+        FOR	${i}	IN	0	5
+         Disable Eth Connection On Port	port=3306
+         Sleep	10s
+         Reset Eth Connection
+         Sleep	10s
+        END
+        ${content}=	Create List	0 events acknowledged
+	${result}=	Find In Log With Timeout	${logCbd}	${start}	${content}	40
+	Should Be True	${result}	msg=No SELECT done by broker in the DB
+        Stop Engine
+        Stop Broker
+
+NetworkDBFailU7
+	[Documentation]	network failure test between broker and database: we wait for the connection to be established and then we shut down the connection for 60s (with unified_sql)
+        [Tags]	Broker	Database	Network	unified_sql
+	Config Engine	${1}
+        Config Broker	central
+	Config Broker Sql Output	central	unified_sql
+        Reset Eth Connection
+	Broker Config Output Set	central	central-broker-unified-sql	db_host	127.0.0.1
+	Broker Config Output set	central	central-broker-unified-sql	connections_count	5
+	Broker Config Log	central	sql	trace
+        Config Broker	rrd
+        Config Broker	module
+	${start}=	Get Current Date
+	Start Broker
+	Start Engine
+	${result}=	Check Connections
+	Should Be True	${result}	msg=Broker and Engine are not connected
+	${content}=	Create List	run query: SELECT
+	${result}=	Find In Log With Timeout	${logCbd}	${start}	${content}	40
+	Should Be True	${result}	msg=No SELECT done by broker in the DB
+        FOR	${i}	IN	0	5
+         Disable Eth Connection On Port	port=3306
+         Sleep	10s
+         Reset Eth Connection
+         Sleep	10s
+        END
+        ${content}=	Create List	0 events acknowledged
+	${result}=	Find In Log With Timeout	${logCbd}	${start}	${content}	40
+	Should Be True	${result}	msg=No SELECT done by broker in the DB
+        Stop Engine
+        Stop Broker
 
 *** Keywords ***
 Disable Sleep Enable
 	[Arguments]		${interval}
 	Disable Eth Connection On Port	port=3306
-	Sleep   		${interval}
+	Sleep			${interval}
 	Reset Eth Connection
+
 Network Failure
-	[Arguments]		${interval}
+	[Arguments]	${interval}
 	Config Engine	${1}
-    Config Broker  	central
-	Broker Config Output Set        central     central-broker-master-sql       db_host     127.0.0.1
-    Broker Config Log				central		sql								trace
-	${start}=		Get Current Date
-	Start mysql
-    Start Broker
+	Config Broker	module
+	Config Broker	rrd
+	Config Broker	central
+	Broker Config Output Set	central	central-broker-master-sql	db_host	127.0.0.1
+	Broker Config Output set	central	central-broker-master-sql	connections_count	5
+	Broker Config Output Set	central	central-broker-master-perfdata	db_host	127.0.0.1
+	Broker Config Output set	central	central-broker-master-perfdata	connections_count	5
+	Broker Config Log	central	sql	trace
+	${start}=	Get Current Date
+	Start Broker
 	Start Engine
-	${log}=			Catenate		SEPARATOR=	${BROKER_LOG}	/central-broker-master.log
-	${content}=		Set Variable	SQL: performing mysql_ping.
-	${result}=		Do When Catch In Log	${log}		${start}	${content}		Disable Sleep Enable	${interval}
-    ${end}=			Get Current Date
-	${content}=		Set Variable	mysql_connection: commit
-	Should Not Be Equal		${result}	False		msg=timeout after 5 minutes (network failure duration : ${interval})
-	${result}=				Catch In Log			${log}		${end}		${content}
-	Should Be True			${result}				msg=timeout after 5 minutes (mysql_ping may be hanging)
+	${content}=	Create List	SQL: performing mysql_ping
+        ${result}=	Find In Log With Timeout	${logCbd}	${start}	${content}	50
+        Should Be True	${result}	msg=We should have a call to mysql_ping every 30s on inactive connections.
+	Disable Sleep Enable	${interval}
+	${end}=	Get Current Date
+	${content}=	Create List	mysql_connection: commit
+	${result}=	Find In Log With Timeout	${logCbd}	${end}	${content}	80
+	Should Be True	${result}	msg=timeout after network to be restablished (network failure duration : ${interval})
 	Stop Broker
 	Stop Engine
-    Stop Mysql
+
+*** Variables ***
+${logCbd}	${BROKER_LOG}/central-broker-master.log

centreon-tests/broker-engine/rrd.robot

@@ -16,6 +16,7 @@ Library	../resources/Common.py
 
 *** Test Cases ***
 BRRDDM1
+	Start Mysql
 	[Documentation]	RRD metric deletion on table metric
 	[Tags]	RRD metric deletion on table metric
 	Config Engine	${1}

centreon-tests/resources/Common.py

@@ -42,7 +42,7 @@ def get_date(d: str):
 
 
 def find_in_log_with_timeout(log: str, date, content, timeout: int):
-    limit = time.time() + TIMEOUT
+    limit = time.time() + timeout
     while time.time() < limit:
         if find_in_log(log, date, content):
             return True
@@ -118,12 +118,12 @@ def stop_mysql():
 
 
 def kill_broker():
-    getoutput("kill -SIGTERM $(ps aux | grep '/usr/sbin/cbwd' | grep -v grep | awk '{print $2}')")
-    getoutput("kill -SIGTERM $(ps aux | grep '/usr/sbin/cbd' | grep -v grep | awk '{print $2}')")
+    getoutput("kill -SIGKILL $(ps aux | grep '/usr/sbin/cbwd' | grep -v grep | awk '{print $2}')")
+    getoutput("kill -SIGKILL $(ps aux | grep '/usr/sbin/cbd' | grep -v grep | awk '{print $2}')")
 
 
 def kill_engine():
-    getoutput("kill -SIGTERM $(ps aux | grep '/usr/sbin/centengine' | grep -v grep | awk '{print $2}')")
+    getoutput("kill -SIGKILL $(ps aux | grep '/usr/sbin/centengine' | grep -v grep | awk '{print $2}')")
 
 
 def clear_retention():

centreon-tests/resources/resources.robot

@@ -32,9 +32,10 @@ Reload Broker
 	Send Signal To Process	SIGHUP	b2
 
 Stop Broker
-	${result}=	Terminate Process	b1
+	Log to console	Extinction of b1
+	${result}=	Terminate Process	b1	kill=False
 	Should Be Equal As Integers	${result.rc}	0
-	${result}=	Terminate Process	b2
+	${result}=	Terminate Process	b2	kill=False
 	Should Be Equal As Integers	${result.rc}	0
 
 Stop Processes
@@ -84,7 +85,7 @@ Check Connections
 	[Return]	${retval}
 
 Disable Eth Connection On Port
-	[Arguments]		${port}
+	[Arguments]	${port}
 	RUN	iptables -A INPUT -p tcp --dport ${port} -j DROP
 	RUN	iptables -A OUTPUT -p tcp --dport ${port} -j DROP
 	RUN	iptables -A FORWARD -p tcp --dport ${port} -j DROP