Merge remote-tracking branch 'server/bug4811-with-bug5288' into bug4811-with-bug5288

java/com.sap.sailing.landscape/src/com/sap/sailing/landscape/impl/SailingAnalyticsProcessImpl.java

@@ -5,6 +5,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.util.Optional;
+import java.util.logging.Logger;
 
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
@@ -19,12 +20,15 @@ import com.jcraft.jsch.SftpException;
 import com.sap.sailing.landscape.SailingAnalyticsMetrics;
 import com.sap.sailing.landscape.SailingAnalyticsProcess;
 import com.sap.sse.common.Duration;
+import com.sap.sse.common.TimePoint;
 import com.sap.sse.landscape.Host;
 import com.sap.sse.landscape.application.impl.ApplicationProcessImpl;
 
 public class SailingAnalyticsProcessImpl<ShardingKey>
 extends ApplicationProcessImpl<ShardingKey, SailingAnalyticsMetrics, SailingAnalyticsProcess<ShardingKey>>
 implements SailingAnalyticsProcess<ShardingKey> {
+    private final static Logger logger = Logger.getLogger(SailingAnalyticsProcessImpl.class.getName());
+
     /**
      * Tries to obtain the port from the {@code env.sh} file found in the {@code serverDirectory}
      */
@@ -58,6 +62,16 @@ implements SailingAnalyticsProcess<ShardingKey> {
     @Override
     public String getServerName(Optional<Duration> optionalTimeout)
             throws JSchException, IOException, InterruptedException, SftpException, ParseException {
-        return getStatus(optionalTimeout).get("servername").toString();
+        String result = null;
+        final TimePoint start = TimePoint.now();
+        while (result == null && optionalTimeout.map(d->start.until(TimePoint.now()).compareTo(d) < 0).orElse(true)) {
+            try {
+                result = getStatus(optionalTimeout).get("servername").toString();
+            } catch (Exception e) {
+                logger.info("Exception waiting for server name."+optionalTimeout.map(d->" Waiting another "+d.minus(start.until(TimePoint.now())).toString()).orElse(""));
+                Thread.sleep(Duration.ONE_SECOND.times(5).asMillis());
+            }
+        }
+        return result;
     }
 }

java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/AwsLandscape.java

@@ -174,15 +174,34 @@ extends Landscape<ShardingKey, MetricsT, ProcessT> {
     void deleteSnapshot(Region region, String snapshotId);
     
     /**
-     * Finds EC2 instances in the {@code region} that have a tag named {@code tagName} with value {@code tagValue}.
+     * Finds EC2 instances in the {@code region} that have a tag named {@code tagName} with value {@code tagValue}. The
+     * result includes instances regardless their state; they are not required to be RUNNING.
+     * 
+     * @see #getRunningHostsWithTagValue(Region, String)
      */
     Iterable<AwsInstance<ShardingKey, MetricsT>> getHostsWithTagValue(Region region, String tagName, String tagValue);
 
     /**
-     * Finds EC2 instances in the {@code region} that have a tag named {@code tagName}. The tag may have any value.
+     * Finds EC2 instances in the {@code region} that have a tag named {@code tagName}. The tag may have any value. The
+     * result includes instances regardless their state; they are not required to be RUNNING.
+     * 
+     * @see #getRunningHostsWithTag(Region, String)
      */
     Iterable<AwsInstance<ShardingKey, MetricsT>> getHostsWithTag(Region region, String tagName);
     
+    /**
+     * Finds EC2 instances in the {@code region} that have a tag named {@code tagName}. The tag may have any value. The
+     * instances returned have been in state RUNNING at the time of the request.
+     */
+    Iterable<AwsInstance<ShardingKey, MetricsT>> getRunningHostsWithTag(Region region, String tagName);
+
+    /**
+     * Finds EC2 instances in the {@code region} that have a tag named {@code tagName} with value {@code tagValue}. The
+     * instances returned have been in state RUNNING at the time of the request.
+     */
+    Iterable<AwsInstance<ShardingKey, MetricsT>> getRunningHostsWithTagValue(Region region, String tagName,
+            String tagValue);
+
     KeyPairInfo getKeyPairInfo(Region region, String keyName);
 
     Iterable<KeyPairInfo> getAllKeyPairInfos(Region region);
@@ -472,4 +491,5 @@ extends Landscape<ShardingKey, MetricsT, ProcessT> {
     Iterable<ApplicationReplicaSet<ShardingKey, MetricsT, ProcessT>> getApplicationReplicaSetsByTag(Region region,
             String tagName, BiFunction<Host, String, ProcessT> processFactoryFromHostAndServerDirectory,
             Optional<Duration> optionalTimeout) throws Exception;
+
 }

java/com.sap.sse.landscape.aws/src/com/sap/sse/landscape/aws/impl/AwsLandscapeImpl.java

@@ -512,14 +512,27 @@ implements AwsLandscape<ShardingKey, MetricsT, ProcessT> {
     @Override
     public Iterable<AwsInstance<ShardingKey, MetricsT>> getHostsWithTagValue(com.sap.sse.landscape.Region region,
             String tagName, String tagValue) {
-        Filter filter = Filter.builder().name("tag:"+tagName).values(tagValue).build();
-        return getHostsWithFilter(region, filter);
+        Filter filter = getHostWithTagValueFilter(tagName, tagValue).build();
+        return getHostsWithFilters(region, filter);
     }
 
-    private Iterable<AwsInstance<ShardingKey, MetricsT>> getHostsWithFilter(com.sap.sse.landscape.Region region,
-            Filter filter) {
+    private Filter.Builder getHostWithTagValueFilter(String tagName, String tagValue) {
+        return Filter.builder().name("tag:"+tagName).values(tagValue);
+    }
+
+    @Override
+    public Iterable<AwsInstance<ShardingKey, MetricsT>> getRunningHostsWithTagValue(com.sap.sse.landscape.Region region,
+            String tagName, String tagValue) {
+        return getHostsWithFilters(region, getRunningHostFilter());
+    }
+    
+    private Filter getRunningHostFilter() {
+        return Filter.builder().name("instance-state-name").values("running").build();
+    }
+
+    private Iterable<AwsInstance<ShardingKey, MetricsT>> getHostsWithFilters(com.sap.sse.landscape.Region region, Filter... filters) {
         final List<AwsInstance<ShardingKey, MetricsT>> result = new ArrayList<>();
-        final DescribeInstancesResponse instanceResponse = getEc2Client(getRegion(region)).describeInstances(b->b.filters(filter));
+        final DescribeInstancesResponse instanceResponse = getEc2Client(getRegion(region)).describeInstances(b->b.filters(filters));
         for (final Reservation r : instanceResponse.reservations()) {
             for (final Instance i : r.instances()) {
                 result.add(getHost(region, i));
@@ -536,10 +549,18 @@ implements AwsLandscape<ShardingKey, MetricsT, ProcessT> {
         return getHost(region, getInstance(instanceId, region));
     }
     
+    @Override
+    public Iterable<AwsInstance<ShardingKey, MetricsT>> getRunningHostsWithTag(com.sap.sse.landscape.Region region, String tagName) {
+        return getHostsWithFilters(region, getFilterForHostWithTag(Filter.builder(), tagName), getRunningHostFilter());
+    }
+    
     @Override
     public Iterable<AwsInstance<ShardingKey, MetricsT>> getHostsWithTag(com.sap.sse.landscape.Region region, String tagName) {
-        Filter filter = Filter.builder().name("tag-key").values(tagName).build();
-        return getHostsWithFilter(region, filter);
+        return getHostsWithFilters(region, getFilterForHostWithTag(Filter.builder(), tagName));
+    }
+    
+    private Filter getFilterForHostWithTag(Filter.Builder builder, String tagName) {
+        return builder.name("tag-key").values(tagName).build();
     }
 
     private Comparator<? super Image> getMachineImageCreationDateComparator() {
@@ -741,7 +762,7 @@ implements AwsLandscape<ShardingKey, MetricsT, ProcessT> {
     @Override
     public ReverseProxyCluster<ShardingKey, MetricsT, ProcessT, RotatingFileBasedLog> getCentralReverseProxy(com.sap.sse.landscape.Region region) {
         ApacheReverseProxyCluster<ShardingKey, MetricsT, ProcessT, RotatingFileBasedLog> reverseProxyCluster = new ApacheReverseProxyCluster<>(this);
-        for (final AwsInstance<ShardingKey, MetricsT> reverseProxyHost : getHostsWithTag(region, CENTRAL_REVERSE_PROXY_TAG_NAME)) {
+        for (final AwsInstance<ShardingKey, MetricsT> reverseProxyHost : getRunningHostsWithTag(region, CENTRAL_REVERSE_PROXY_TAG_NAME)) {
             reverseProxyCluster.addHost(reverseProxyHost);
         }
         return reverseProxyCluster;
@@ -901,7 +922,7 @@ implements AwsLandscape<ShardingKey, MetricsT, ProcessT> {
     }
 
     private Iterable<AwsInstance<ShardingKey, MetricsT>> getMongoDBHosts(com.sap.sse.landscape.Region region) {
-        return getHostsWithTag(region, MONGO_REPLICA_SETS_TAG_NAME);
+        return getRunningHostsWithTag(region, MONGO_REPLICA_SETS_TAG_NAME);
     }
 
     /**
@@ -943,7 +964,7 @@ implements AwsLandscape<ShardingKey, MetricsT, ProcessT> {
     @Override
     public RabbitMQEndpoint getDefaultRabbitConfiguration(AwsRegion region) {
         final RabbitMQEndpoint result;
-        final Iterable<AwsInstance<ShardingKey, MetricsT>> rabbitMQHostsInRegion = getHostsWithTag(region, RABBITMQ_TAG_NAME);
+        final Iterable<AwsInstance<ShardingKey, MetricsT>> rabbitMQHostsInRegion = getRunningHostsWithTag(region, RABBITMQ_TAG_NAME);
         if (rabbitMQHostsInRegion.iterator().hasNext()) {
             final AwsInstance<ShardingKey, MetricsT> anyRabbitMQHost = rabbitMQHostsInRegion.iterator().next();
             result = new RabbitMQEndpoint() {
@@ -985,7 +1006,7 @@ implements AwsLandscape<ShardingKey, MetricsT, ProcessT> {
     public Iterable<ApplicationProcessHost<ShardingKey, MetricsT, ProcessT>> getApplicationProcessHostsByTag(com.sap.sse.landscape.Region region, String tagName,
             BiFunction<Host, String, ProcessT> processFactoryFromHostAndServerDirectory) {
         final List<ApplicationProcessHost<ShardingKey, MetricsT, ProcessT>> result = new ArrayList<>();
-        for (final AwsInstance<ShardingKey, MetricsT> host : getHostsWithTag(region, tagName)) {
+        for (final AwsInstance<ShardingKey, MetricsT> host : getRunningHostsWithTag(region, tagName)) {
             final ApplicationProcessHost<ShardingKey, MetricsT, ProcessT> applicationProcessHost =
                     new ApplicationProcessHostImpl<ShardingKey, MetricsT, ProcessT>(
                             host.getInstanceId(), host.getAvailabilityZone(), this, processFactoryFromHostAndServerDirectory);

java/com.sap.sse.landscape/src/com/sap/sse/landscape/application/impl/ApplicationProcessImpl.java

@@ -6,7 +6,6 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.net.URL;
 import java.util.Optional;
-import java.util.logging.Level;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -23,6 +22,7 @@ import com.jcraft.jsch.ChannelSftp;
 import com.jcraft.jsch.JSchException;
 import com.jcraft.jsch.SftpException;
 import com.sap.sse.common.Duration;
+import com.sap.sse.common.TimePoint;
 import com.sap.sse.landscape.DefaultProcessConfigurationVariables;
 import com.sap.sse.landscape.Host;
 import com.sap.sse.landscape.ProcessConfigurationVariable;
@@ -182,26 +182,30 @@ implements ApplicationProcess<ShardingKey, MetricsT, ProcessT> {
     }
 
     @Override
-    public String getMasterServerName(Optional<Duration> optionalTimeout) throws ClientProtocolException, IOException, ParseException {
-        final JSONObject replicationStatus = getReplicationStatus(optionalTimeout);
+    public String getMasterServerName(Optional<Duration> optionalTimeout) throws ClientProtocolException, IOException, ParseException, InterruptedException {
         String result = null;
-        for (final Object replicable : (JSONArray) replicationStatus.get("replicables")) {
-            final JSONObject replicableAsJson = (JSONObject) replicable;
-            if (replicableAsJson.get("replicatedfrom") != null) {
-                final JSONObject replicatedFrom = (JSONObject) replicableAsJson.get("replicatedfrom");
-                final String hostname = replicatedFrom.get("hostname").toString();
-                final int port = ((Number) replicatedFrom.get("port")).intValue();
-                // the bearer token that is good for the connection to this process instance should then also
-                // be recognized as valid on this instance's master and have the READ_REPLICATOR permission for the same server name
-                try {
-                    final JSONObject masterReplicationStatus = getReplicationStatus(optionalTimeout, hostname, port);
-                    result = masterReplicationStatus.get("servername").toString();
-                    break;
-                } catch (Exception e) {
-                    // could, e.g., be a security issue with a master with a different servername where there bearer token leads
-                    // to a user who does not have the READ_REPLICATOR permission for that server; log and ignore:
-                    logger.log(Level.INFO, "Problem trying to read replication status on "+hostname+":"+port+" to obtain server name; ignoring", e);
+        boolean obtainedReplicationStatusSuccessfully = false;
+        final TimePoint start = TimePoint.now();
+        while (!obtainedReplicationStatusSuccessfully && optionalTimeout.map(d->start.until(TimePoint.now()).compareTo(d) < 0).orElse(true)) {
+            try {
+                final JSONObject replicationStatus = getReplicationStatus(optionalTimeout);
+                obtainedReplicationStatusSuccessfully = true;
+                for (final Object replicable : (JSONArray) replicationStatus.get("replicables")) {
+                    final JSONObject replicableAsJson = (JSONObject) replicable;
+                    if (replicableAsJson.get("replicatedfrom") != null) {
+                        final JSONObject replicatedFrom = (JSONObject) replicableAsJson.get("replicatedfrom");
+                        final String hostname = replicatedFrom.get("hostname").toString();
+                        final int port = ((Number) replicatedFrom.get("port")).intValue();
+                        // the bearer token that is good for the connection to this process instance should then also
+                        // be recognized as valid on this instance's master and have the READ_REPLICATOR permission for the same server name
+                        final JSONObject masterReplicationStatus = getReplicationStatus(optionalTimeout, hostname, port);
+                        result = masterReplicationStatus.get("servername").toString();
+                        break;
+                    }
                 }
+            } catch (Exception e) {
+                logger.info("Exception waiting for server name."+optionalTimeout.map(d->" Waiting another "+d.minus(start.until(TimePoint.now())).toString()).orElse(""));
+                Thread.sleep(Duration.ONE_SECOND.times(5).asMillis());
             }
         }
         return result;

java/com.sap.sse.replication/resources/shiro.ini

@@ -39,7 +39,11 @@ securityManager.subjectFactory=$webSubjectFactoryWithAnonymousPrincipalSupport
 # authentication with POST parameters "username" and "password".
 bearerToken = com.sap.sse.security.BearerTokenOrBasicOrFormAuthenticationFilter
 
+# The same as bearerToken, but with additional fallback to anonymous authentication.
+# In this case sufficient permission checks have to be done on the application side.
+bearerTokenOrAnonymous = com.sap.sse.security.BearerTokenOrBasicOrFormOrAnonymousAuthenticationFilter
+
 # Specifying filter chains for URL patterns
 
 [urls]
-/replication = bearerToken
+/replication = bearerTokenOrAnonymous