Fix a number of bugs in raft, and it can now go for around 1400 simulations before hitting a race condition

src/io/v3/simulator_handle.hpp

@@ -236,12 +236,17 @@ class SimulatorHandle {
         }
       }
 
-      bool all_servers_blocked = blocked_servers < server_addresses_.size();
+      bool all_servers_blocked = blocked_servers == server_addresses_.size();
 
       if (all_servers_blocked) {
         return;
       }
 
+      std::cout << "only " << (int)blocked_servers << " servers blocked, but size is " << (int)server_addresses_.size()
+                << std::endl;
+
+      // __asm__ __volatile__("yield");
+
       cv_.wait(lock);
     }
   }
@@ -267,18 +272,17 @@ class SimulatorHandle {
       return false;
     }
 
-    if (in_flight_.empty()) {
-      // std::cout << "returning from tick: empty in_flight_" << std::endl;
-      return false;
-    }
-
-    // TODO this is not deterministic time advancement
     // clock ticks forwards by this many microseconds on average
     std::poisson_distribution<> time_distrib(100);
     uint64_t clock_advance = time_distrib(rng_);
     cluster_wide_time_microseconds_ += clock_advance;
 
-    // std::cout << "looking at message in tick" << std::endl;
+    cv_.notify_all();
+
+    if (in_flight_.empty()) {
+      // return early here because there are no messages to schedule
+      return false;
+    }
 
     if (config_.scramble_messages) {
       // scramble messages
@@ -318,8 +322,6 @@ class SimulatorHandle {
       om_vec->second.emplace_back(std::move(opaque_message));
     }
 
-    cv_.notify_all();
-
     return true;
   }
 
@@ -382,7 +384,7 @@ class SimulatorHandle {
       bool made_progress = MaybeTickSimulator();
       lock.lock();
       if (!should_shut_down_ && !made_progress) {
-        std::cout << "waiting on cv" << std::endl;
+        // std::cout << "waiting on cv" << std::endl;
         cv_.wait(lock);
       }
       blocked_on_receive_ -= 1;

tests/simulation/raft.cpp

@@ -608,7 +608,7 @@ void RunSimulation() {
     cli_req.opaque_data = std::vector<uint8_t>{1, 2, 3, 4};
 
     ResponseFuture<ReplicationResponse> response_future =
-        cli_io.RequestWithTimeout<ReplicationRequest, ReplicationResponse>(srv_addr_3, cli_req, 100);
+        cli_io.RequestWithTimeout<ReplicationRequest, ReplicationResponse>(leader, cli_req, 100);
 
     // receive response
     ResponseResult<ReplicationResponse> response_result = response_future.Wait();
@@ -622,6 +622,9 @@ void RunSimulation() {
 
     if (response.retry_leader) {
       leader = response.retry_leader.value();
+      std::cout << "client redirected to leader server " << leader.last_known_port << std::endl;
+    } else {
+      std::cout << "client NOT redirected to leader server " << std::endl;
     }
   }
 
@@ -642,7 +645,7 @@ int main() {
   int n_tests = 500;
 
   for (int i = 0; i < n_tests; i++) {
-    std::cout << "========================== NEW SIMULATION ==========================" << std::endl;
+    std::cout << "========================== NEW SIMULATION " << i << " ==========================" << std::endl;
     RunSimulation();
   }