From 68579d1c1112fdc96cd39ef8e59f0f5a3ea3b864 Mon Sep 17 00:00:00 2001 From: BrennanConroy Date: Thu, 7 Feb 2019 13:54:13 -0800 Subject: [PATCH] C++ client: Skip flaky tests (#7358) --- .../connection_impl_tests.cpp | 3 +- .../hub_connection_impl_tests.cpp | 446 +++++++++--------- .../websocket_transport_tests.cpp | 3 +- 3 files changed, 229 insertions(+), 223 deletions(-) diff --git a/src/SignalR/clients/cpp/test/signalrclienttests/connection_impl_tests.cpp b/src/SignalR/clients/cpp/test/signalrclienttests/connection_impl_tests.cpp index 54cd737383..b74382c6bb 100644 --- a/src/SignalR/clients/cpp/test/signalrclienttests/connection_impl_tests.cpp +++ b/src/SignalR/clients/cpp/test/signalrclienttests/connection_impl_tests.cpp @@ -768,7 +768,8 @@ TEST(connection_impl_stop, can_start_and_stop_connection) ASSERT_EQ(_XPLATSTR("[state change] disconnecting -> disconnected\n"), remove_date_from_log_entry(log_entries[3])); } -TEST(connection_impl_stop, can_start_and_stop_connection_multiple_times) +// Flaky test: "transport timed out when trying to connect" +TEST(connection_impl_stop, DISABLED_can_start_and_stop_connection_multiple_times) { auto writer = std::shared_ptr{std::make_shared()}; diff --git a/src/SignalR/clients/cpp/test/signalrclienttests/hub_connection_impl_tests.cpp b/src/SignalR/clients/cpp/test/signalrclienttests/hub_connection_impl_tests.cpp index 767b6ad81a..3c2dfd6e27 100644 --- a/src/SignalR/clients/cpp/test/signalrclienttests/hub_connection_impl_tests.cpp +++ b/src/SignalR/clients/cpp/test/signalrclienttests/hub_connection_impl_tests.cpp @@ -85,7 +85,8 @@ TEST(stop, disconnected_callback_called_when_hub_connection_stops) ASSERT_TRUE(disconnected_invoked); } -TEST(stop, connection_stopped_when_going_out_of_scope) +// Flaky test: ASSERT_EQ(4U, log_entries.size()) was 3 +TEST(stop, DISABLED_connection_stopped_when_going_out_of_scope) { std::shared_ptr writer(std::make_shared()); @@ -223,7 +224,8 @@ TEST(hub_invocation, hub_connection_invokes_users_code_on_hub_invocations) ASSERT_EQ(_XPLATSTR("[\"message\",1]"), *payload); } -TEST(invoke, invoke_creates_correct_payload) +// Flaky test: "error" thrown in test body +TEST(invoke, DISABLED_invoke_creates_correct_payload) { utility::string_t payload; @@ -250,7 +252,8 @@ TEST(invoke, invoke_creates_correct_payload) ASSERT_EQ(_XPLATSTR("{\"arguments\":[],\"target\":\"method\",\"type\":1}\x1e"), payload); } -TEST(invoke, callback_not_called_if_send_throws) +// "error" thrown in test body +TEST(invoke, DISABLED_callback_not_called_if_send_throws) { auto websocket_client = create_test_websocket_client( /* receive function */ []() { return pplx::task_from_result(std::string("{ }\x1e")); }, @@ -269,7 +272,7 @@ TEST(invoke, callback_not_called_if_send_throws) ASSERT_STREQ("error", e.what()); } - // stop completes all outstanting callbacks so if we did not remove a callback when `invoke_void` failed an + // stop completes all outstanding callbacks so if we did not remove a callback when `invoke_void` failed an // unobserved exception exception would be thrown. Note that this would happen on a different thread and would // crash the process hub_connection->stop().get(); @@ -434,7 +437,8 @@ TEST(invoke_void, invoke_unblocks_task_when_server_completes_call) ASSERT_TRUE(true); } -TEST(invoke_void, invoke_logs_if_callback_for_given_id_not_found) +// Flaky test: crashes test process +TEST(invoke_void, DISABLED_invoke_logs_if_callback_for_given_id_not_found) { auto message_received_event = std::make_shared(); @@ -473,230 +477,230 @@ TEST(invoke_void, invoke_logs_if_callback_for_given_id_not_found) } // TODO Flaky until hub_connection.start waits for handshake response -//TEST(invoke_void, invoke_propagates_errors_from_server_as_exceptions) -//{ -// auto callback_registered_event = std::make_shared(); -// -// int call_number = -1; -// auto websocket_client = create_test_websocket_client( -// /* receive function */ [call_number, callback_registered_event]() -// mutable { -// std::string responses[] -// { -// "{ }\x1e", -// "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" -// "{}" -// }; -// -// call_number = std::min(call_number + 1, 2); -// -// if (call_number > 0) -// { -// callback_registered_event->wait(); -// } -// -// return pplx::task_from_result(responses[call_number]); -// }); -// -// auto hub_connection = create_hub_connection(websocket_client); -// try -// { -// hub_connection->start() -// .then([hub_connection, callback_registered_event]() -// { -// auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); -// callback_registered_event->set(); -// return t; -// }).get(); -// -// ASSERT_TRUE(false); // exception expected but not thrown -// } -// catch (const std::runtime_error& e) -// { -// ASSERT_STREQ("\"Ooops\"", e.what()); -// } -//} +TEST(invoke_void, DISABLED_invoke_propagates_errors_from_server_as_exceptions) +{ + auto callback_registered_event = std::make_shared(); + + int call_number = -1; + auto websocket_client = create_test_websocket_client( + /* receive function */ [call_number, callback_registered_event]() + mutable { + std::string responses[] + { + "{ }\x1e", + "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" + "{}" + }; + + call_number = std::min(call_number + 1, 2); + + if (call_number > 0) + { + callback_registered_event->wait(); + } + + return pplx::task_from_result(responses[call_number]); + }); + + auto hub_connection = create_hub_connection(websocket_client); + try + { + hub_connection->start() + .then([hub_connection, callback_registered_event]() + { + auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); + callback_registered_event->set(); + return t; + }).get(); + + ASSERT_TRUE(false); // exception expected but not thrown + } + catch (const std::runtime_error& e) + { + ASSERT_STREQ("\"Ooops\"", e.what()); + } +} // TODO Flaky until hub_connection.start waits for handshake response -//TEST(invoke_void, invoke_propagates_hub_errors_from_server_as_hub_exceptions) -//{ -// auto callback_registered_event = std::make_shared(); -// -// int call_number = -1; -// auto websocket_client = create_test_websocket_client( -// /* receive function */ [call_number, callback_registered_event]() -// mutable { -// std::string responses[] -// { -// "{ }\x1e", -// "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" -// "{}" -// }; -// -// call_number = std::min(call_number + 1, 2); -// -// if (call_number > 0) -// { -// callback_registered_event->wait(); -// } -// -// return pplx::task_from_result(responses[call_number]); -// }); -// -// auto hub_connection = create_hub_connection(websocket_client); -// try -// { -// hub_connection->start() -// .then([hub_connection, callback_registered_event]() -// { -// auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); -// callback_registered_event->set(); -// return t; -// }).get(); -// -// ASSERT_TRUE(false); // exception expected but not thrown -// } -// catch (const hub_exception& e) -// { -// ASSERT_STREQ("\"Ooops\"", e.what()); -// } -//} +TEST(invoke_void, DISABLED_invoke_propagates_hub_errors_from_server_as_hub_exceptions) +{ + auto callback_registered_event = std::make_shared(); + + int call_number = -1; + auto websocket_client = create_test_websocket_client( + /* receive function */ [call_number, callback_registered_event]() + mutable { + std::string responses[] + { + "{ }\x1e", + "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" + "{}" + }; + + call_number = std::min(call_number + 1, 2); + + if (call_number > 0) + { + callback_registered_event->wait(); + } + + return pplx::task_from_result(responses[call_number]); + }); + + auto hub_connection = create_hub_connection(websocket_client); + try + { + hub_connection->start() + .then([hub_connection, callback_registered_event]() + { + auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); + callback_registered_event->set(); + return t; + }).get(); + + ASSERT_TRUE(false); // exception expected but not thrown + } + catch (const hub_exception& e) + { + ASSERT_STREQ("\"Ooops\"", e.what()); + } +} // TODO Flaky until hub_connection.start waits for handshake response -//TEST(invoke_void, invoke_creates_hub_exception_even_if_no_error_data) -//{ -// auto callback_registered_event = std::make_shared(); -// -// int call_number = -1; -// auto websocket_client = create_test_websocket_client( -// /* receive function */ [call_number, callback_registered_event]() -// mutable { -// std::string responses[] -// { -// "{ }\x1e", -// "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" -// "{}" -// }; -// -// call_number = std::min(call_number + 1, 2); -// -// if (call_number > 0) -// { -// callback_registered_event->wait(); -// } -// -// return pplx::task_from_result(responses[call_number]); -// }); -// -// auto hub_connection = create_hub_connection(websocket_client); -// try -// { -// hub_connection->start() -// .then([hub_connection, callback_registered_event]() -// { -// auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); -// callback_registered_event->set(); -// return t; -// }).get(); -// -// ASSERT_TRUE(false); // exception expected but not thrown -// } -// catch (const hub_exception& e) -// { -// ASSERT_STREQ("\"Ooops\"", e.what()); -// } -//} +TEST(invoke_void, DISABLED_invoke_creates_hub_exception_even_if_no_error_data) +{ + auto callback_registered_event = std::make_shared(); + + int call_number = -1; + auto websocket_client = create_test_websocket_client( + /* receive function */ [call_number, callback_registered_event]() + mutable { + std::string responses[] + { + "{ }\x1e", + "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" + "{}" + }; + + call_number = std::min(call_number + 1, 2); + + if (call_number > 0) + { + callback_registered_event->wait(); + } + + return pplx::task_from_result(responses[call_number]); + }); + + auto hub_connection = create_hub_connection(websocket_client); + try + { + hub_connection->start() + .then([hub_connection, callback_registered_event]() + { + auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); + callback_registered_event->set(); + return t; + }).get(); + + ASSERT_TRUE(false); // exception expected but not thrown + } + catch (const hub_exception& e) + { + ASSERT_STREQ("\"Ooops\"", e.what()); + } +} // TODO Flaky until hub_connection.start waits for handshake response -//TEST(invoke_void, invoke_creates_runtime_error_when_hub_exception_indicator_false) -//{ -// auto callback_registered_event = std::make_shared(); -// -// int call_number = -1; -// auto websocket_client = create_test_websocket_client( -// /* receive function */ [call_number, callback_registered_event]() -// mutable { -// std::string responses[] -// { -// "{ }\x1e", -// "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" -// "{}" -// }; -// -// call_number = std::min(call_number + 1, 2); -// -// if (call_number > 0) -// { -// callback_registered_event->wait(); -// } -// -// return pplx::task_from_result(responses[call_number]); -// }); -// -// auto hub_connection = create_hub_connection(websocket_client); -// try -// { -// hub_connection->start() -// .then([hub_connection, callback_registered_event]() -// { -// auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); -// callback_registered_event->set(); -// return t; -// }).get(); -// -// ASSERT_TRUE(false); // exception expected but not thrown -// } -// catch (const signalr_exception& e) -// { -// ASSERT_STREQ("\"Ooops\"", e.what()); -// ASSERT_TRUE(dynamic_cast(&e) == nullptr); -// } -//} +TEST(invoke_void, DISABLED_invoke_creates_runtime_error_when_hub_exception_indicator_false) +{ + auto callback_registered_event = std::make_shared(); + + int call_number = -1; + auto websocket_client = create_test_websocket_client( + /* receive function */ [call_number, callback_registered_event]() + mutable { + std::string responses[] + { + "{ }\x1e", + "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" + "{}" + }; + + call_number = std::min(call_number + 1, 2); + + if (call_number > 0) + { + callback_registered_event->wait(); + } + + return pplx::task_from_result(responses[call_number]); + }); + + auto hub_connection = create_hub_connection(websocket_client); + try + { + hub_connection->start() + .then([hub_connection, callback_registered_event]() + { + auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); + callback_registered_event->set(); + return t; + }).get(); + + ASSERT_TRUE(false); // exception expected but not thrown + } + catch (const signalr_exception& e) + { + ASSERT_STREQ("\"Ooops\"", e.what()); + ASSERT_TRUE(dynamic_cast(&e) == nullptr); + } +} // TODO Flaky until hub_connection.start waits for handshake response -//TEST(invoke_void, invoke_creates_runtime_error) -//{ -// auto callback_registered_event = std::make_shared(); -// -// int call_number = -1; -// auto websocket_client = create_test_websocket_client( -// /* receive function */ [call_number, callback_registered_event]() -// mutable { -// std::string responses[] -// { -// "{ }\x1e", -// "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" -// }; -// -// call_number = std::min(call_number + 1, 1); -// -// if (call_number > 0) -// { -// callback_registered_event->wait(); -// } -// -// return pplx::task_from_result(responses[call_number]); -// }); -// -// auto hub_connection = create_hub_connection(websocket_client); -// try -// { -// hub_connection->start() -// .then([hub_connection, callback_registered_event]() -// { -// auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); -// callback_registered_event->set(); -// return t; -// }).get(); -// -// ASSERT_TRUE(false); // exception expected but not thrown -// } -// catch (const signalr_exception& e) -// { -// ASSERT_STREQ("\"Ooops\"", e.what()); -// ASSERT_TRUE(dynamic_cast(&e) == nullptr); -// } -//} +TEST(invoke_void, DISABLED_invoke_creates_runtime_error) +{ + auto callback_registered_event = std::make_shared(); + + int call_number = -1; + auto websocket_client = create_test_websocket_client( + /* receive function */ [call_number, callback_registered_event]() + mutable { + std::string responses[] + { + "{ }\x1e", + "{ \"type\": 3, \"invocationId\": \"0\", \"error\": \"Ooops\" }\x1e" + }; + + call_number = std::min(call_number + 1, 1); + + if (call_number > 0) + { + callback_registered_event->wait(); + } + + return pplx::task_from_result(responses[call_number]); + }); + + auto hub_connection = create_hub_connection(websocket_client); + try + { + hub_connection->start() + .then([hub_connection, callback_registered_event]() + { + auto t = hub_connection->invoke_void(_XPLATSTR("method"), json::value::array()); + callback_registered_event->set(); + return t; + }).get(); + + ASSERT_TRUE(false); // exception expected but not thrown + } + catch (const signalr_exception& e) + { + ASSERT_STREQ("\"Ooops\"", e.what()); + ASSERT_TRUE(dynamic_cast(&e) == nullptr); + } +} TEST(connection_id, can_get_connection_id) { diff --git a/src/SignalR/clients/cpp/test/signalrclienttests/websocket_transport_tests.cpp b/src/SignalR/clients/cpp/test/signalrclienttests/websocket_transport_tests.cpp index c62ccf673e..8cb2a7893c 100644 --- a/src/SignalR/clients/cpp/test/signalrclienttests/websocket_transport_tests.cpp +++ b/src/SignalR/clients/cpp/test/signalrclienttests/websocket_transport_tests.cpp @@ -97,7 +97,8 @@ TEST(websocket_transport_connect, connect_logs_exceptions) entry); } -TEST(websocket_transport_connect, cannot_call_connect_on_already_connected_transport) +// Flaky test: crashes test process +TEST(websocket_transport_connect, DISABLED_cannot_call_connect_on_already_connected_transport) { auto client = std::make_shared(); auto ws_transport = websocket_transport::create([&](){ return client; }, logger(std::make_shared(), trace_level::none),