Disable idna tracker hostnames by default

include/libtorrent/http_connection.hpp

@@ -73,6 +73,7 @@ using http_handler = std::function<void(error_code const&
 using http_connect_handler = std::function<void(http_connection&)>;
 
 using http_filter_handler = std::function<void(http_connection&, std::vector<tcp::endpoint>&)>;
+using hostname_filter_handler = std::function<bool(http_connection&, string_view)>;
 
 // when bottled, the last two arguments to the handler
 // will always be 0
@@ -86,6 +87,7 @@ struct TORRENT_EXTRA_EXPORT http_connection
 		, int max_bottled_buffer_size
 		, http_connect_handler const& ch
 		, http_filter_handler const& fh
+		, hostname_filter_handler const& hfh
 #ifdef TORRENT_USE_OPENSSL
 		, ssl::context* ssl_ctx
 #endif
@@ -178,6 +180,7 @@ struct TORRENT_EXTRA_EXPORT http_connection
 	http_handler m_handler;
 	http_connect_handler m_connect_handler;
 	http_filter_handler m_filter_handler;
+	hostname_filter_handler m_hostname_filter_handler;
 	deadline_timer m_timer;
 
 	time_duration m_completion_timeout;

include/libtorrent/http_tracker_connection.hpp

@@ -73,6 +73,7 @@ namespace libtorrent {
 		}
 
 		void on_filter(http_connection& c, std::vector<tcp::endpoint>& endpoints);
+		bool on_filter_hostname(http_connection& c, string_view hostname);
 		void on_connect(http_connection& c);
 		void on_response(error_code const& ec, http_parser const& parser
 			, span<char const> data);

include/libtorrent/parse_url.hpp

@@ -39,6 +39,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <string>
 
 #include "libtorrent/error_code.hpp"
+#include "libtorrent/string_view.hpp"
 
 namespace libtorrent {
 
@@ -50,6 +51,10 @@ namespace libtorrent {
 	// split a URL in its base and path parts
 	TORRENT_EXTRA_EXPORT std::tuple<std::string, std::string>
 		split_url(std::string url, error_code& ec);
+
+	// returns true if the hostname contains any IDNA (internationalized domain
+	// name) labels.
+	TORRENT_EXTRA_EXPORT bool is_idna(string_view hostname);
 }
 
 #endif

include/libtorrent/settings_pack.hpp

@@ -845,6 +845,12 @@ namespace aux {
 			// tracker request to loopback will be ignored.
 			tracker_ssrf_mitigation,
 
+			// when disabled, any tracker or web seed with an IDNA hostname
+			// (internationalized domain name) is ignored. This is a security
+			// precaution to avoid various unicode encoding attacks that might
+			// happen at the application level.
+			allow_idna,
+
 			max_bool_setting_internal
 		};
 

simulation/test_http_connection.cpp

@@ -184,6 +184,7 @@ std::shared_ptr<http_connection> test_request(io_service& ios
 			std::printf("CONNECTED: %s\n", url.c_str());
 		}
 		, lt::http_filter_handler()
+		, lt::hostname_filter_handler()
 #ifdef TORRENT_USE_OPENSSL
 		, &ssl_ctx
 #endif
@@ -655,6 +656,7 @@ TORRENT_TEST(http_connection_ssl_proxy)
 		}
 		, true, 1024*1024, lt::http_connect_handler()
 		, http_filter_handler()
+		, hostname_filter_handler()
 #ifdef TORRENT_USE_OPENSSL
 		, &ssl_ctx
 #endif

simulation/test_tracker.cpp

@@ -309,6 +309,16 @@ struct sim_config : sim::default_config
 				result.push_back(address_v6::from_string("::1"));
 			return duration_cast<chrono::high_resolution_clock::duration>(chrono::milliseconds(1));
 		}
+		if (hostname == "xn--tracker-.com")
+		{
+			result.push_back(address_v4::from_string("123.0.0.2"));
+			return duration_cast<chrono::high_resolution_clock::duration>(chrono::milliseconds(100));
+		}
+		if (hostname == "redirector.com")
+		{
+			result.push_back(address_v4::from_string("123.0.0.4"));
+			return duration_cast<chrono::high_resolution_clock::duration>(chrono::milliseconds(100));
+		}
 
 		return default_config::hostname_lookup(requestor, hostname, result, ec);
 	}
@@ -605,14 +615,16 @@ TORRENT_TEST(ipv6_support_bind_v6_v4)
 // port 8080.
 template <typename Setup, typename Announce, typename Test1, typename Test2>
 void tracker_test(Setup setup, Announce a, Test1 test1, Test2 test2
-	, char const* url_path = "/announce")
+	, char const* url_path = "/announce"
+	, char const* redirect = "http://123.0.0.2/announce")
 {
 	using sim::asio::ip::address_v4;
 	sim_config network_cfg;
 	sim::simulation sim{network_cfg};
 
 	sim::asio::io_service tracker_ios(sim, address_v4::from_string("123.0.0.2"));
 	sim::asio::io_service tracker_ios6(sim, address_v6::from_string("ff::dead:beef"));
+	sim::asio::io_service redirector_ios(sim, address_v4::from_string("123.0.0.4"));
 
 	sim::asio::io_service tracker_lo_ios(sim, address_v4::from_string("127.0.0.1"));
 	sim::asio::io_service tracker_lo_ios6(sim, address_v6::from_string("::1"));
@@ -622,11 +634,13 @@ void tracker_test(Setup setup, Announce a, Test1 test1, Test2 test2
 	sim::http_server http6(tracker_ios6, 8080);
 	sim::http_server http_lo(tracker_lo_ios, 8080);
 	sim::http_server http6_lo(tracker_lo_ios6, 8080);
+	sim::http_server http_redirect(redirector_ios, 8080);
 
 	http.register_handler(url_path, a);
 	http6.register_handler(url_path, a);
 	http_lo.register_handler(url_path, a);
 	http6_lo.register_handler(url_path, a);
+	http_redirect.register_redirect(url_path, redirect);
 
 	lt::session_proxy zombie;
 
@@ -1380,6 +1394,48 @@ TORRENT_TEST(tracker_ssrf_IPv6)
 	test_ssrf("/unusual-announce-path", false, "http://[::1]:8080/unusual-announce-path", true);
 }
 
+bool test_idna(char const* tracker_url, char const* redirect
+	, bool const feature_on)
+{
+	bool got_announce = false;
+	tracker_test(
+		[&](lt::add_torrent_params& p, lt::session& ses)
+		{
+			settings_pack pack;
+			pack.set_bool(settings_pack::allow_idna, feature_on);
+			ses.apply_settings(pack);
+			p.trackers.emplace_back(tracker_url);
+			return 60;
+		},
+		[&](std::string method, std::string req
+			, std::map<std::string, std::string>& headers)
+		{
+			got_announce = true;
+			return sim::send_response(200, "OK", 11) + "d5:peers0:e";
+		}
+		, [](torrent_handle h) {}
+		, [](torrent_handle h) {}
+		, "/announce"
+		, redirect ? redirect : ""
+		);
+	return got_announce;
+}
+
+TORRENT_TEST(tracker_idna)
+{
+	TEST_EQUAL(test_idna("http://tracker.com:8080/announce", nullptr, true), true);
+	TEST_EQUAL(test_idna("http://tracker.com:8080/announce", nullptr, false), true);
+
+	TEST_EQUAL(test_idna("http://xn--tracker-.com:8080/announce", nullptr, true), true);
+	TEST_EQUAL(test_idna("http://xn--tracker-.com:8080/announce", nullptr, false), false);
+}
+
+TORRENT_TEST(tracker_idna_redirect)
+{
+	TEST_EQUAL(test_idna("http://redirector.com:8080/announce", "http://xn--tracker-.com:8080/announce", true), true);
+	TEST_EQUAL(test_idna("http://redirector.com:8080/announce", "http://xn--tracker-.com:8080/announce", false), false);
+}
+
 // This test sets up two peers, one seed an one downloader. The downloader has
 // two trackers, both in tier 0. The behavior we expect is that it picks one of
 // the trackers at random and announces to it. Since both trackers are working,

simulation/test_web_seed.cpp

@@ -98,6 +98,43 @@ add_torrent_params create_torrent(file_storage& fs, bool const pad_files = false
 	ret.save_path = ".";
 	return ret;
 }
+
+struct sim_config : sim::default_config
+{
+	explicit sim_config() {}
+
+	chrono::high_resolution_clock::duration hostname_lookup(
+		asio::ip::address const& requestor
+		, std::string hostname
+		, std::vector<asio::ip::address>& result
+		, boost::system::error_code& ec) override
+	{
+		auto const ret = duration_cast<chrono::high_resolution_clock::duration>(chrono::milliseconds(100));
+		if (hostname == "2.server.com")
+		{
+			result.push_back(address_v4::from_string("2.2.2.2"));
+			return ret;
+		}
+		if (hostname == "2.xn--server-.com")
+		{
+			result.push_back(address_v4::from_string("2.2.2.2"));
+			return ret;
+		}
+		if (hostname == "3.server.com")
+		{
+			result.push_back(address_v4::from_string("3.3.3.3"));
+			return ret;
+		}
+		if (hostname == "3.xn--server-.com")
+		{
+			result.push_back(address_v4::from_string("3.3.3.3"));
+			return ret;
+		}
+
+		return default_config::hostname_lookup(requestor, hostname, result, ec);
+	}
+};
+
 // this is the general template for these tests. create the session with custom
 // settings (Settings), set up the test, by adding torrents with certain
 // arguments (Setup), run the test and verify the end state (Test)
@@ -108,7 +145,7 @@ void run_test(Setup const& setup
 	, lt::seconds const timeout = lt::seconds{100})
 {
 	// setup the simulation
-	sim::default_config network_cfg;
+	sim_config network_cfg;
 	sim::simulation sim{network_cfg};
 	std::unique_ptr<sim::asio::io_service> ios = make_io_service(sim, 0);
 	lt::session_proxy zombie;
@@ -634,3 +671,78 @@ TORRENT_TEST(web_seed_connection_limit)
 	TEST_CHECK(std::accumulate(expected.begin(), expected.end(), 0) == 2);
 }
 
+bool test_idna(char const* url, char const* redirect, bool allow_idna)
+{
+	using namespace lt;
+	file_storage fs;
+	fs.add_file("1", 0xc030);
+	lt::add_torrent_params params = ::create_torrent(fs);
+	params.url_seeds.emplace_back(url);
+
+	bool seeding = false;
+
+	error_code ignore;
+	remove("1", ignore);
+
+	run_test(
+		[&](lt::session& ses)
+		{
+			settings_pack pack;
+			pack.set_bool(settings_pack::allow_idna, allow_idna);
+			ses.apply_settings(pack);
+			ses.async_add_torrent(params);
+		},
+		[&](lt::session&, lt::alert const* alert) {
+			if (lt::alert_cast<lt::torrent_finished_alert>(alert))
+				seeding = true;
+		},
+		[&](sim::simulation& sim, lt::session&)
+		{
+			// http1 is the root web server that will just redirect requests to
+			// other servers
+			sim::asio::io_service web_server1(sim, address_v4::from_string("2.2.2.2"));
+			sim::http_server http1(web_server1, 8080);
+			// redirect file 1 and file 2 to the same servers
+			if (redirect)
+				http1.register_redirect("/1", redirect);
+
+			// server for serving the content
+			sim::asio::io_service web_server2(sim, address_v4::from_string("3.3.3.3"));
+			sim::http_server http2(web_server2, 8080);
+			serve_content_for(http2, "/1", fs, file_index_t(0));
+
+			sim.run();
+		}
+	);
+
+	return seeding;
+}
+
+TORRENT_TEST(idna)
+{
+	// disallow IDNA hostnames
+	TEST_EQUAL(test_idna("http://3.server.com:8080", nullptr, false), true);
+	TEST_EQUAL(test_idna("http://3.xn--server-.com:8080", nullptr, false), false);
+
+	// allow IDNA hostnames
+	TEST_EQUAL(test_idna("http://3.server.com:8080", nullptr, true), true);
+	TEST_EQUAL(test_idna("http://3.xn--server-.com:8080", nullptr, true), true);
+}
+
+TORRENT_TEST(idna_redirect)
+{
+	// disallow IDNA hostnames
+	TEST_EQUAL(test_idna("http://2.server.com:8080", "http://3.server.com:8080/1", false), true);
+	TEST_EQUAL(test_idna("http://2.server.com:8080", "http://3.xn--server-.com:8080/1", false), false);
+
+	TEST_EQUAL(test_idna("http://2.xn--server-.com:8080", "http://3.server.com:8080/1", false), false);
+	TEST_EQUAL(test_idna("http://2.xn--server-.com:8080", "http://3.xn--server-.com:8080/1", false), false);
+
+	// allow IDNA hostnames
+	TEST_EQUAL(test_idna("http://2.server.com:8080", "http://3.server.com:8080/1", true), true);
+	TEST_EQUAL(test_idna("http://2.server.com:8080", "http://3.xn--server-.com:8080/1", true), true);
+
+	TEST_EQUAL(test_idna("http://2.xn--server-.com:8080", "http://3.server.com:8080/1", true), true);
+	TEST_EQUAL(test_idna("http://2.xn--server-.com:8080", "http://3.xn--server-.com:8080/1", true), true);
+}
+

src/http_connection.cpp

@@ -66,6 +66,7 @@ http_connection::http_connection(io_service& ios
 	, int max_bottled_buffer_size
 	, http_connect_handler const& ch
 	, http_filter_handler const& fh
+	, hostname_filter_handler const& hfh
 #ifdef TORRENT_USE_OPENSSL
 	, ssl::context* ssl_ctx
 #endif
@@ -82,6 +83,7 @@ http_connection::http_connection(io_service& ios
 	, m_handler(handler)
 	, m_connect_handler(ch)
 	, m_filter_handler(fh)
+	, m_hostname_filter_handler(hfh)
 	, m_timer(ios)
 	, m_completion_timeout(seconds(5))
 	, m_limiter_timer(ios)
@@ -141,6 +143,14 @@ void http_connection::get(std::string const& url, time_duration timeout, int pri
 		return;
 	}
 
+	if (m_hostname_filter_handler && !m_hostname_filter_handler(*this, hostname))
+	{
+		ec.assign(errors::banned_by_ip_filter, libtorrent_category());
+		lt::get_io_service(m_timer).post(std::bind(&http_connection::callback
+			, me, ec, span<char>{}));
+		return;
+	}
+
 	if (protocol != "http"
 #ifdef TORRENT_USE_OPENSSL
 		&& protocol != "https"

src/http_tracker_connection.cpp

@@ -208,6 +208,7 @@ namespace libtorrent {
 			, true, settings.get_int(settings_pack::max_http_recv_buffer_size)
 			, std::bind(&http_tracker_connection::on_connect, shared_from_this(), _1)
 			, std::bind(&http_tracker_connection::on_filter, shared_from_this(), _1, _2)
+			, std::bind(&http_tracker_connection::on_filter_hostname, shared_from_this(), _1, _2)
 #ifdef TORRENT_USE_OPENSSL
 			, tracker_req().ssl_ctx
 #endif
@@ -350,6 +351,15 @@ namespace libtorrent {
 			fail(errors::banned_by_ip_filter);
 	}
 
+	// returns true if the hostname is allowed
+	bool http_tracker_connection::on_filter_hostname(http_connection&
+		, string_view hostname)
+	{
+		aux::session_settings const& settings = m_man.settings();
+		if (settings.get_bool(settings_pack::allow_idna)) return true;
+		return !is_idna(hostname);
+	}
+
 	void http_tracker_connection::on_connect(http_connection& c)
 	{
 		error_code ec;

src/parse_url.cpp

@@ -174,4 +174,20 @@ namespace libtorrent {
 		return std::make_tuple(std::move(base), std::move(path));
 	}
 
+	TORRENT_EXTRA_EXPORT bool is_idna(string_view hostname)
+	{
+		for (;;)
+		{
+			auto dot = hostname.find('.');
+			string_view const label = (dot == string_view::npos) ? hostname : hostname.substr(0, dot);
+			if (label.size() >= 4
+				&& (label[0] == 'x' || label[0] == 'X')
+				&& (label[1] == 'n' || label[1] == 'N')
+				&& label.substr(2, 2) == "--"_sv)
+				return true;
+			if (dot == string_view::npos) return false;
+			hostname = hostname.substr(dot + 1);
+		}
+	}
+
 }

src/settings_pack.cpp

@@ -212,6 +212,7 @@ constexpr int CLOSE_FILE_INTERVAL = 0;
 		SET(piece_extent_affinity, false, nullptr),
 		SET(validate_https_trackers, false, &session_impl::update_validate_https),
 		SET(tracker_ssrf_mitigation, true, nullptr),
+		SET(allow_idna, false, nullptr),
 	}});
 
 	aux::array<int_setting_entry_t, settings_pack::num_int_settings> const int_settings