Skip to content

Commit e34dcd6

Browse files
ggreenwayggreenway
authored
Fix crash in tcp_proxy (#4323)
* Fix crash in tcp_proxy. Closing the upstream connection is not safe from the Filter destructor, because it triggers events back into the downstream connection, which is partially destructed. Ensure that the upstream connection is closed before the destructor is called. Fixes #4310 Signed-off-by: Greg Greenway <ggreenway@apple.com>
1 parent ae6a252 commit e34dcd6

File tree

5 files changed

+67
-14
lines changed

5 files changed

+67
-14
lines changed

‎source/common/tcp_proxy/tcp_proxy.cc‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -135,13 +135,8 @@ Filter::~Filter() {
135135
access_log->log(nullptr, nullptr, nullptr, getRequestInfo());
136136
}
137137

138-
if (upstream_handle_) {
139-
upstream_handle_->cancel();
140-
}
141-
142-
if (upstream_conn_data_) {
143-
upstream_conn_data_->connection().close(Network::ConnectionCloseType::NoFlush);
144-
}
138+
ASSERT(upstream_handle_ == nullptr);
139+
ASSERT(upstream_conn_data_ == nullptr);
145140
}
146141

147142
TcpProxyStats Config::SharedConfig::generateStats(Stats::Scope& scope) {
@@ -412,17 +407,29 @@ void Filter::onDownstreamEvent(Network::ConnectionEvent event) {
412407
if (event == Network::ConnectionEvent::RemoteClose) {
413408
upstream_conn_data_->connection().close(Network::ConnectionCloseType::FlushWrite);
414409

415-
if (upstream_conn_data_ != nullptr &&
416-
upstream_conn_data_->connection().state() != Network::Connection::State::Closed) {
417-
config_->drainManager().add(config_->sharedConfig(), std::move(upstream_conn_data_),
418-
std::move(upstream_callbacks_), std::move(idle_timer_),
419-
read_callbacks_->upstreamHost());
410+
// Events raised from the previous line may cause upstream_conn_data_ to be NULL if
411+
// it was able to immediately flush all data.
412+
413+
if (upstream_conn_data_ != nullptr) {
414+
if (upstream_conn_data_->connection().state() != Network::Connection::State::Closed) {
415+
config_->drainManager().add(config_->sharedConfig(), std::move(upstream_conn_data_),
416+
std::move(upstream_callbacks_), std::move(idle_timer_),
417+
read_callbacks_->upstreamHost());
418+
} else {
419+
upstream_conn_data_.reset();
420+
}
420421
}
421422
} else if (event == Network::ConnectionEvent::LocalClose) {
422423
upstream_conn_data_->connection().close(Network::ConnectionCloseType::NoFlush);
423424
upstream_conn_data_.reset();
424425
disableIdleTimer();
425426
}
427+
} else if (upstream_handle_) {
428+
if (event == Network::ConnectionEvent::LocalClose ||
429+
event == Network::ConnectionEvent::RemoteClose) {
430+
upstream_handle_->cancel();
431+
upstream_handle_ = nullptr;
432+
}
426433
}
427434
}
428435

‎test/common/http/conn_manager_impl_test.cc‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1705,6 +1705,10 @@ TEST_F(HttpConnectionManagerImplTest, WebSocketPrefixAndAutoHostRewrite) {
17051705
Buffer::OwnedImpl fake_input("1234");
17061706
conn_manager_->onData(fake_input, false);
17071707

1708+
Tcp::ConnectionPool::UpstreamCallbacks* upstream_callbacks = nullptr;
1709+
EXPECT_CALL(*conn_pool_.connection_data_, addUpstreamCallbacks(_))
1710+
.WillOnce(
1711+
Invoke([&](Tcp::ConnectionPool::UpstreamCallbacks& cb) { upstream_callbacks = &cb; }));
17081712
conn_pool_.host_->hostname_ = "newhost";
17091713
conn_pool_.poolReady(upstream_conn_);
17101714

@@ -1714,6 +1718,7 @@ TEST_F(HttpConnectionManagerImplTest, WebSocketPrefixAndAutoHostRewrite) {
17141718
EXPECT_EQ(1U, stats_.named_.downstream_cx_websocket_total_.value());
17151719
EXPECT_EQ(0U, stats_.named_.downstream_cx_http1_active_.value());
17161720

1721+
upstream_callbacks->onEvent(Network::ConnectionEvent::RemoteClose);
17171722
filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList();
17181723
conn_manager_.reset();
17191724
EXPECT_EQ(0U, stats_.named_.downstream_cx_websocket_active_.value());
@@ -1753,8 +1758,13 @@ TEST_F(HttpConnectionManagerImplTest, WebSocketEarlyData) {
17531758
EXPECT_CALL(upstream_conn_, write(_, false));
17541759
EXPECT_CALL(upstream_conn_, write(BufferEqual(&early_data), false));
17551760
EXPECT_CALL(filter_callbacks_.connection_, readDisable(false));
1761+
Tcp::ConnectionPool::UpstreamCallbacks* upstream_callbacks = nullptr;
1762+
EXPECT_CALL(*conn_pool_.connection_data_, addUpstreamCallbacks(_))
1763+
.WillOnce(
1764+
Invoke([&](Tcp::ConnectionPool::UpstreamCallbacks& cb) { upstream_callbacks = &cb; }));
17561765
conn_pool_.poolReady(upstream_conn_);
17571766

1767+
upstream_callbacks->onEvent(Network::ConnectionEvent::RemoteClose);
17581768
filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList();
17591769
conn_manager_.reset();
17601770
}
@@ -1828,7 +1838,12 @@ TEST_F(HttpConnectionManagerImplTest, WebSocketEarlyEndStream) {
18281838

18291839
EXPECT_CALL(upstream_conn_, write(_, false));
18301840
EXPECT_CALL(upstream_conn_, write(_, true)).Times(0);
1841+
Tcp::ConnectionPool::UpstreamCallbacks* upstream_callbacks = nullptr;
1842+
EXPECT_CALL(*conn_pool_.connection_data_, addUpstreamCallbacks(_))
1843+
.WillOnce(
1844+
Invoke([&](Tcp::ConnectionPool::UpstreamCallbacks& cb) { upstream_callbacks = &cb; }));
18311845
conn_pool_.poolReady(upstream_conn_);
1846+
upstream_callbacks->onEvent(Network::ConnectionEvent::RemoteClose);
18321847
filter_callbacks_.connection_.dispatcher_.clearDeferredDeleteList();
18331848
conn_manager_.reset();
18341849
}

‎test/common/network/filter_manager_impl_test.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,8 @@ TEST_F(NetworkFilterManagerTest, RateLimitAndTcpProxy) {
214214
EXPECT_CALL(upstream_connection, write(BufferEqual(&buffer), _));
215215
read_buffer_.add("hello");
216216
manager.onRead();
217+
218+
connection.raiseEvent(ConnectionEvent::RemoteClose);
217219
}
218220

219221
} // namespace Network

‎test/common/tcp_proxy/tcp_proxy_test.cc‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,12 @@ class TcpProxyTest : public testing::Test {
348348
.WillByDefault(SaveArg<0>(&access_log_data_));
349349
}
350350

351+
~TcpProxyTest() {
352+
if (filter_ != nullptr) {
353+
filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
354+
}
355+
}
356+
351357
void configure(const envoy::config::filter::network::tcp_proxy::v2::TcpProxy& config) {
352358
config_.reset(new Config(config, factory_context_));
353359
}
@@ -734,6 +740,22 @@ TEST_F(TcpProxyTest, DisconnectBeforeData) {
734740
filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
735741
}
736742

743+
// Test that if the downstream connection is closed before the upstream connection
744+
// is established, the upstream connection is cancelled.
745+
TEST_F(TcpProxyTest, RemoteClosetBeforeUpstreamConnected) {
746+
setup(1);
747+
EXPECT_CALL(*conn_pool_handles_.at(0), cancel());
748+
filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
749+
}
750+
751+
// Test that if the downstream connection is closed before the upstream connection
752+
// is established, the upstream connection is cancelled.
753+
TEST_F(TcpProxyTest, LocalClosetBeforeUpstreamConnected) {
754+
setup(1);
755+
EXPECT_CALL(*conn_pool_handles_.at(0), cancel());
756+
filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::LocalClose);
757+
}
758+
737759
TEST_F(TcpProxyTest, UpstreamConnectFailure) {
738760
setup(1, accessLogConfig("%RESPONSE_FLAGS%"));
739761

@@ -873,6 +895,7 @@ TEST_F(TcpProxyTest, IdleTimeoutWithOutstandingDataFlushed) {
873895
TEST_F(TcpProxyTest, AccessLogUpstreamHost) {
874896
setup(1, accessLogConfig("%UPSTREAM_HOST% %UPSTREAM_CLUSTER%"));
875897
raiseEventUpstreamConnected(0);
898+
filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
876899
filter_.reset();
877900
EXPECT_EQ(access_log_data_, "127.0.0.1:80 fake_cluster");
878901
}
@@ -881,6 +904,7 @@ TEST_F(TcpProxyTest, AccessLogUpstreamHost) {
881904
TEST_F(TcpProxyTest, AccessLogUpstreamLocalAddress) {
882905
setup(1, accessLogConfig("%UPSTREAM_LOCAL_ADDRESS%"));
883906
raiseEventUpstreamConnected(0);
907+
filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
884908
filter_.reset();
885909
EXPECT_EQ(access_log_data_, "2.2.2.2:50000");
886910
}
@@ -893,6 +917,7 @@ TEST_F(TcpProxyTest, AccessLogDownstreamAddress) {
893917
filter_callbacks_.connection_.remote_address_ =
894918
Network::Utility::resolveUrl("tcp://1.1.1.1:40000");
895919
setup(1, accessLogConfig("%DOWNSTREAM_REMOTE_ADDRESS_WITHOUT_PORT% %DOWNSTREAM_LOCAL_ADDRESS%"));
920+
filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
896921
filter_.reset();
897922
EXPECT_EQ(access_log_data_, "1.1.1.1 1.1.1.2:20000");
898923
}
@@ -1075,6 +1100,9 @@ TEST_F(TcpProxyRoutingTest, NonRoutableConnection) {
10751100

10761101
EXPECT_EQ(total_cx + 1, config_->stats().downstream_cx_total_.value());
10771102
EXPECT_EQ(non_routable_cx + 1, config_->stats().downstream_cx_no_route_.value());
1103+
1104+
// Cleanup
1105+
filter_callbacks_.connection_.raiseEvent(Network::ConnectionEvent::RemoteClose);
10781106
}
10791107

10801108
TEST_F(TcpProxyRoutingTest, RoutableConnection) {
@@ -1087,7 +1115,8 @@ TEST_F(TcpProxyRoutingTest, RoutableConnection) {
10871115
connection_.local_address_ = std::make_shared<Network::Address::Ipv4Instance>("1.2.3.4", 9999);
10881116

10891117
// Expect filter to try to open a connection to specified cluster.
1090-
EXPECT_CALL(factory_context_.cluster_manager_, tcpConnPoolForCluster("fake_cluster", _, _));
1118+
EXPECT_CALL(factory_context_.cluster_manager_, tcpConnPoolForCluster("fake_cluster", _, _))
1119+
.WillOnce(Return(nullptr));
10911120

10921121
filter_->onNewConnection();
10931122

‎test/integration/fake_upstream.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -536,7 +536,7 @@ AssertionResult FakeRawConnection::write(const std::string& data, bool end_strea
536536
Network::FilterStatus FakeRawConnection::ReadFilter::onData(Buffer::Instance& data,
537537
bool end_stream) {
538538
Thread::LockGuard lock(parent_.lock_);
539-
ENVOY_LOG(debug, "got {} bytes", data.length());
539+
ENVOY_LOG(debug, "got {} bytes, end_stream {}", data.length(), end_stream);
540540
parent_.data_.append(data.toString());
541541
parent_.half_closed_ = end_stream;
542542
data.drain(data.length());

0 commit comments

Comments
 (0)