extended file storage service with properties and metadata

java/com.sap.sse.filestorage/META-INF/MANIFEST.MF

@@ -13,4 +13,5 @@ Require-Bundle: org.apache.servicemix.bundles.aws-java-sdk;bundle-version="1.9.8
  com.fasterxml.jackson.core.jackson-databind,
  com.fasterxml.jackson.core.jackson-core,
  com.fasterxml.jackson.core.jackson-annotations,
- org.apache.commons.logging;bundle-version="1.1.3"
+ org.apache.commons.logging;bundle-version="1.1.3",
+ com.sap.sse.common

java/com.sap.sse.filestorage/src/com/sap/sse/filestorage/FileStorageService.java

@@ -5,26 +5,48 @@ import java.io.InputStream;
 import java.net.URI;
 
 /**
- * Service for storing files. The interface is intentionally agnostic of the underlying implementation,
- * which may be provided e.g. by using Amazon's S3 or simply saving the files to a statically reachable
- * directory on one of our servers.
- * Implementations of this service are announced via the OSGi service registry.
+ * Service for storing files. The interface is intentionally agnostic of the underlying implementation, which may be
+ * provided e.g. by using Amazon's S3 or simply saving the files to a statically reachable directory on one of our
+ * servers. Implementations of this service are announced via the OSGi service registry.
+ * 
+ * TODO The storage service to use should be configured via the AdminConsole. Upon auto-discovering the available
+ * services, the AdminConsole should allow to edit properties for each service (e.g. access credentials for AWS S3). We
+ * probably need a generic property discovery mechanism (key-value based?), and to survive server restarts these
+ * properties should be saved to the MongoDB.
  * 
- * TODO The storage service to use should be configured via the AdminConsole. Upon auto-discovering the
- * available services, the AdminConsole should allow to edit properties for each service (e.g. access
- * credentials for AWS S3). We probably need a generic property discovery mechanism (key-value based?),
- * and to survive server restarts these properties should be saved to the MongoDB.
  * @author Fredrik Teschke
  *
  */
 public interface FileStorageService {
     /**
-     * @param originalFileName may be {@code null}
+     * @param originalFileName
+     *            may be {@code null}
      */
-    URI storeFile(InputStream is, String originalFileName, long lengthInBytes) throws IOException;
-    
+    URI storeFile(InputStream is, String originalFileName, long lengthInBytes) throws IOException,
+            OperationFailedException, InvalidPropertiesException;
+
     /**
      * From the given {@code uri} it should be possible to determine the file to remove.
      */
-    void removeFile(URI uri);
+    void removeFile(URI uri) throws OperationFailedException, InvalidPropertiesException;
+
+    Property[] getProperties();
+
+    /**
+     * @throws IllegalArgumentException
+     *             if {@code name} is not a valid property name
+     */
+    void setProperty(String name, String value) throws IllegalArgumentException;
+
+    String getName();
+
+    /**
+     * Description text that explains the storage service.
+     */
+    String getDescription();
+
+    /**
+     * Test whether properties are valid, e.g. by trying to log in using access credentials provided as properties.
+     */
+    void testProperties() throws InvalidPropertiesException;
 }

java/com.sap.sse.filestorage/src/com/sap/sse/filestorage/InvalidPropertiesException.java

@@ -0,0 +1,44 @@
+package com.sap.sse.filestorage;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.sap.sse.common.Util.Pair;
+
+public class InvalidPropertiesException extends Exception {
+    private static final long serialVersionUID = -7328897153875728802L;
+    private final Map<Property, String> perPropertyMessages = new HashMap<>();
+
+    public InvalidPropertiesException(String message) {
+        super(message);
+    }
+
+    public InvalidPropertiesException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidPropertiesException(String message,
+            @SuppressWarnings("unchecked") Pair<Property, String>... perPropertyMessages) {
+        this(message, null, perPropertyMessages);
+    }
+
+    public InvalidPropertiesException(String message, Throwable cause,
+            @SuppressWarnings("unchecked") Pair<Property, String>... perPropertyMessages) {
+        super(message, cause);
+        for (Pair<Property, String> pair : perPropertyMessages) {
+            this.perPropertyMessages.put(pair.getA(), pair.getB());
+        }
+    }
+
+    @Override
+    /**
+     * Returns the overall error message (always exists).
+     */
+    public String getMessage() {
+        return super.getMessage();
+    }
+
+    public Map<Property, String> getPerPropertyMessage() {
+        return perPropertyMessages;
+    }
+}

java/com.sap.sse.filestorage/src/com/sap/sse/filestorage/OperationFailedException.java

@@ -0,0 +1,9 @@
+package com.sap.sse.filestorage;
+
+public class OperationFailedException extends Exception {
+    private static final long serialVersionUID = 3361800673687196602L;
+    
+    public OperationFailedException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

java/com.sap.sse.filestorage/src/com/sap/sse/filestorage/Property.java

@@ -0,0 +1,17 @@
+package com.sap.sse.filestorage;
+
+/**
+ * Property of a file storage service.
+ * @author Fredrik Teschke
+ *
+ */
+public interface Property {
+    boolean isRequired();
+    String getName();
+    
+    /**
+     * @return {@code null} if not yet initialized
+     */
+    String getValue();
+    String getDescription();
+}

java/com.sap.sse.filestorage/src/com/sap/sse/filestorage/impl/AmazonS3FileStorageServiceImpl.java

@@ -3,99 +3,166 @@ package com.sap.sse.filestorage.impl;
 import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.UUID;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import com.amazonaws.AmazonClientException;
 import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.profile.ProfileCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.CannedAccessControlList;
 import com.amazonaws.services.s3.model.DeleteObjectRequest;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.PutObjectRequest;
 import com.sap.sse.filestorage.FileStorageService;
+import com.sap.sse.filestorage.InvalidPropertiesException;
+import com.sap.sse.filestorage.OperationFailedException;
+import com.sap.sse.filestorage.Property;
 
 /**
- * For testing purposes configure the access credentials as follows:
- * To link this service to an AWS account, create the following file: ~/.aws/credentials
- * and add credentials to it (get the access id and secret key from
+ * For testing purposes configure the access credentials as follows: To link this service to an AWS account, create the
+ * following file: ~/.aws/credentials and add credentials to it (get the access id and secret key from
  * https://console.aws.amazon.com/iam/home?#security_credential).
  * 
- * TODO configure credentials in AdminConsole
- * TODO configure bucket name in AdminConsole
+ * TODO configure credentials in AdminConsole TODO configure bucket name in AdminConsole
  * 
  * @author Fredrik Teschke
  *
  */
 public class AmazonS3FileStorageServiceImpl implements FileStorageService {
     private static final Logger logger = Logger.getLogger(AmazonS3FileStorageServiceImpl.class.getName());
-    
+
     private static final String baseUrl = "s3.amazonaws.com";
     private static final String retrievalProtocol = "http";
-    private static final String bucketName = "ftes-sap-sailing";
-    private final AmazonS3 s3;
-    
+    // private static final String bucketName = "ftes-sap-sailing";
+
+    private final PropertyImpl accessId = new PropertyImpl("accessId", true, "Access ID");
+    private final PropertyImpl accessKey = new PropertyImpl("accessKey", true, "Secret Access Key");
+    private final PropertyImpl bucketName = new PropertyImpl("bucketName", true,
+            "Name of Bucket to use (has to already exist)");
+    private final Map<String, PropertyImpl> properties = new HashMap<>();
+
     public AmazonS3FileStorageServiceImpl() {
-        /*
-         * The ProfileCredentialsProvider will return your [default]
-         * credential profile by reading from the credentials file located at
-         * (~/.aws/credentials).
-         */
+        addProperties(accessId, accessKey, bucketName);
+    }
+
+    private void addProperties(PropertyImpl... properties) {
+        for (PropertyImpl p : properties) {
+            this.properties.put(p.getName(), p);
+        }
+    }
+
+    private void testCredentials(AWSCredentials credentials) throws Exception {
+        AmazonS3Client s3 = new AmazonS3Client(credentials);
+        s3.getS3AccountOwner(); // might throw exception
+    }
+
+    private AmazonS3Client createS3Client() throws InvalidPropertiesException {
         AWSCredentials credentials = null;
-        try {
-            credentials = new ProfileCredentialsProvider().getCredentials();
-        } catch (Exception e) {
-            throw new AmazonClientException(
-                    "Cannot load the credentials from the credential profiles file. " +
-                    "Please make sure that your credentials file is at the correct " +
-                    "location (~/.aws/credentials), and is in valid format.",
-                    e);
+
+        // first try to use properties
+        if (accessId.getValue() != null && accessKey.getValue() != null) {
+            credentials = new BasicAWSCredentials(accessId.getValue(), accessKey.getValue());
+
+            try {
+                testCredentials(credentials);
+                return new AmazonS3Client(credentials);
+            } catch (Exception e) {
+                throw new InvalidPropertiesException("Access ID and secret key and ID seem to be invalid", e);
+            }
         }
 
-        s3 = new AmazonS3Client(credentials);
+        // if properties are empty, read credentials from ~/.aws/credentials
+        try {
+            credentials = new ProfileCredentialsProvider().getCredentials();
+            testCredentials(credentials);
+            return new AmazonS3Client(credentials);
+        } catch (Exception e) {
+            throw new InvalidPropertiesException(
+                    "Credentials in ~/.aws/credentials seem to be invalid (tried this as fallback because properties were empty)",
+                    e);
+        }
     }
-    
+
     private static String getKey(String originalFileName) {
         String key = UUID.randomUUID().toString();
         if (originalFileName != null) {
             String ending = originalFileName.substring(originalFileName.lastIndexOf("."));
             key += ending;
-//            key += "/" + originalFileName;
+            // key += "/" + originalFileName;
         }
         return key;
     }
-    
-    private static URI getUri(String key) {
+
+    private URI getUri(String key) {
         try {
-            return new URI(retrievalProtocol, bucketName + "." + baseUrl, "/" + key, null);
+            return new URI(retrievalProtocol, bucketName.getValue() + "." + baseUrl, "/" + key, null);
         } catch (URISyntaxException e) {
             logger.log(Level.WARNING, "Could not create URI for uploaded file with key " + key, e);
             return null;
         }
     }
 
-
     @Override
-    public URI storeFile(InputStream is, String originalFileName, long lengthInBytes) {
+    public URI storeFile(InputStream is, String originalFileName, long lengthInBytes)
+            throws InvalidPropertiesException, OperationFailedException {
         ObjectMetadata metadata = new ObjectMetadata();
         metadata.setContentLength(lengthInBytes);
         String key = getKey(originalFileName);
-        PutObjectRequest request = new PutObjectRequest(bucketName, key, is, metadata)
-            .withCannedAcl(CannedAccessControlList.PublicRead);
-        s3.putObject(request);
+        PutObjectRequest request = new PutObjectRequest(bucketName.getValue(), key, is, metadata)
+                .withCannedAcl(CannedAccessControlList.PublicRead);
+        AmazonS3Client s3Client = createS3Client();
+        try {
+            s3Client.putObject(request);
+        } catch (AmazonClientException e) {
+            throw new OperationFailedException("Could not store file " + originalFileName, e);
+        }
         URI uri = getUri(key);
         logger.info("Stored file " + uri);
         return uri;
     }
 
     @Override
-    public void removeFile(URI uri) {
-        String key = uri.getPath().substring(1); //remove initial slash
-        s3.deleteObject(new DeleteObjectRequest(bucketName, key));
+    public void removeFile(URI uri) throws InvalidPropertiesException, OperationFailedException {
+        String key = uri.getPath().substring(1); // remove initial slash
+        AmazonS3Client s3Client = createS3Client();
+        try {
+            s3Client.deleteObject(new DeleteObjectRequest(bucketName.getValue(), key));
+        } catch (AmazonClientException e) {
+            throw new OperationFailedException("Could not remove file " + uri.toString(), e);
+        }
         logger.info("Removed file " + uri);
     }
 
+    @Override
+    public Property[] getProperties() {
+        return new Property[] { accessId, accessKey, bucketName };
+    }
+
+    @Override
+    public void setProperty(String name, String value) {
+        if (!properties.containsKey(name)) {
+            throw new IllegalArgumentException("Property " + name + " does not exist");
+        }
+        properties.get(name).setValue(value);
+    }
+
+    @Override
+    public String getName() {
+        return "Amazon S3";
+    }
+
+    @Override
+    public String getDescription() {
+        return "Store files on Amazon S3. The resulting file name is a random UUID plus the original file ending.";
+    }
+
+    @Override
+    public void testProperties() throws InvalidPropertiesException {
+        createS3Client();
+    }
 }

java/com.sap.sse.filestorage/src/com/sap/sse/filestorage/impl/PropertyImpl.java

@@ -0,0 +1,45 @@
+package com.sap.sse.filestorage.impl;
+
+import com.sap.sse.filestorage.Property;
+
+public class PropertyImpl implements Property {
+    private final boolean isRequired;
+    private final String name;
+    private final String description;
+    private String value;
+    
+    public PropertyImpl(String name, boolean isRequired, String description) {
+        this(name, isRequired, description, null);
+    }
+    
+    public PropertyImpl(String name, boolean isRequired, String description, String value) {
+        this.name = name;
+        this.description = description;
+        this.value = value;
+        this.isRequired = isRequired;
+    }
+
+    @Override
+    public boolean isRequired() {
+        return isRequired;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public String getValue() {
+        return value;
+    }
+
+    @Override
+    public String getDescription() {
+        return description;
+    }
+
+    public void setValue(String value) {
+        this.value = value;
+    }
+}