Created test that proves that Replicator can recover from connection problems

java/com.sap.sailing.server.replication/src/com/sap/sailing/server/replication/impl/ReplicationServiceImpl.java

@@ -5,6 +5,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
+import java.net.ConnectException;
 import java.net.URL;
 import java.net.URLConnection;
 import java.util.HashMap;
@@ -91,10 +92,20 @@ public class ReplicationServiceImpl implements ReplicationService, OperationExec
         this.exchangeName = exchangeName;
     }
     
-    private Channel createChannel(String exchangeName) throws IOException {
+    private Channel createMasterChannel(String exchangeName) throws IOException {
         final ConnectionFactory connectionFactory = new ConnectionFactory();
         connectionFactory.setHost("localhost"); // ...and use default port
-        Channel result = connectionFactory.newConnection().createChannel();
+        
+        Channel result = null;
+        try {
+            result = connectionFactory.newConnection().createChannel();
+        } catch (ConnectException ex) {
+            // make sure to log something meaningful
+            logger.severe("Could not connect to messaging queue on " + connectionFactory.getHost() + ":" + connectionFactory.getPort() + "/" + exchangeName);
+            throw ex;
+        }
+        
+        logger.info("Connected to " + connectionFactory.getHost() + ":" + connectionFactory.getPort() + "/" + exchangeName);
         result.exchangeDeclare(exchangeName, "fanout");
         return result;
     }
@@ -116,7 +127,7 @@ public class ReplicationServiceImpl implements ReplicationService, OperationExec
             addAsListenerToRacingEventService();
             synchronized (this) {
                 if (masterChannel == null) {
-                    masterChannel = createChannel(exchangeName);
+                    masterChannel = createMasterChannel(exchangeName);
                 }
             }
         }

java/com.sap.sailing.server.replication/src/com/sap/sailing/server/replication/impl/Replicator.java

@@ -8,6 +8,7 @@ import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import com.rabbitmq.client.AlreadyClosedException;
 import com.rabbitmq.client.QueueingConsumer;
 import com.rabbitmq.client.QueueingConsumer.Delivery;
 import com.rabbitmq.client.ShutdownSignalException;
@@ -82,6 +83,13 @@ public class Replicator implements Runnable {
                         new ByteArrayInputStream(bytesFromMessage));
                 RacingEventServiceOperation<?> operation = (RacingEventServiceOperation<?>) ois.readObject();
                 applyOrQueue(operation);
+            } catch (AlreadyClosedException ace) {
+                // ignore this exception and try again after some time
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
             } catch (ShutdownSignalException sse) {
                 logger.info("Received "+sse.getMessage()+". Terminating "+this);
                 break;