diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/module-info.java b/endorsed/src/org.apache.sis.cloud.aws/main/module-info.java index 5ab60c1c7a7..7fc1f857a0b 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/module-info.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/module-info.java @@ -19,7 +19,8 @@ * Java NIO wrappers for Amazon Simple Storage Service (S3). * * @author Martin Desruisseaux (Geomatys) - * @version 1.4 + * @author Quentin Bialota (Geomatys) + * @version 1.7 * @since 1.2 */ module org.apache.sis.cloud.aws { diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java index b8aa19d3eb3..73d0468c409 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystem.java @@ -16,6 +16,8 @@ */ package org.apache.sis.cloud.aws.s3; +import java.net.URI; +import java.net.URISyntaxException; import java.util.Set; import java.util.Collections; import java.util.regex.PatternSyntaxException; @@ -33,6 +35,7 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -46,6 +49,7 @@ * which is kept ready-to-use until the file system is {@linkplain #close closed}. * * @author Martin Desruisseaux (Geomatys) + * @author Quentin Bialota (Geomatys) */ final class ClientFileSystem extends FileSystem { /** @@ -59,6 +63,22 @@ final class ClientFileSystem extends FileSystem { */ final String accessKey; + /** + * The S3 host (if not stored on Amazon Infrastructure), or {@code null} if none. + */ + final String host; + + /** + * The S3 port (if not stored on Amazon Infrastructure), or {@code -1} if none. + */ + final int port; + + /** + * Whether the S3 HTTP Protocol is secure (if this information is not stored on Amazon Infrastructure). + * Default is {@code true}. + */ + final boolean isHttps; + /** * The provider of this file system. */ @@ -82,12 +102,15 @@ final class ClientFileSystem extends FileSystem { final String duplicatedSeparator; /** - * Creates a file system with default credential and default separator. + * Creates a file system with default hostname and default separator. */ - ClientFileSystem(final FileService provider, final S3Client client) { + ClientFileSystem(final FileService provider, final S3Client client, String accessKey) { this.provider = provider; this.client = client; - this.accessKey = null; + this.accessKey = accessKey; + this.host = null; + this.port = -1; + this.isHttps = true; this.separator = DEFAULT_SEPARATOR; duplicatedSeparator = DEFAULT_SEPARATOR + DEFAULT_SEPARATOR; } @@ -97,12 +120,16 @@ final class ClientFileSystem extends FileSystem { * * @param provider the provider creating this file system. * @param region the AWS region, or {@code null} for default. + * @param host the host or {@code null} for AWS request. + * @param port the port or {@code -1} for AWS request. + * @param isHttps the protocol is secure or not or {@code null} for AWS request. * @param accessKey the AWS S3 access key for this file system. * @param secret the password. * @param separator the separator in paths, or {@code null} for the default value. */ - ClientFileSystem(final FileService provider, final Region region, final String accessKey, final String secret, - String separator) + ClientFileSystem(final FileService provider, final Region region, final String host, final int port, + final boolean isHttps, final String accessKey, final String secret, + String separator) throws URISyntaxException { if (separator == null) { separator = DEFAULT_SEPARATOR; @@ -110,11 +137,22 @@ final class ClientFileSystem extends FileSystem { ArgumentChecks.ensureNonEmpty("separator", separator); this.provider = provider; this.accessKey = accessKey; - S3ClientBuilder builder = S3Client.builder().credentialsProvider( - StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secret))); + S3ClientBuilder builder = S3Client.builder(); + if (secret != null) { + builder = builder.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secret))); + } if (region != null) { builder = builder.region(region); } + this.host = host; + this.port = port; + this.isHttps = isHttps; + if (host != null) { + String hostname = (port < 0 ? host + ':' + port : host); + String protocol = (this.isHttps ? "https" : "http"); + builder = builder.endpointOverride(new URI(protocol + "://" + hostname)) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()); + } client = builder.build(); this.separator = separator; duplicatedSeparator = separator.concat(separator); @@ -148,7 +186,7 @@ public synchronized void close() throws IOException { final S3Client c = client; client = null; if (c != null) try { - provider.dispose(accessKey); + provider.dispose(new ClientFileSystemKey(accessKey, host, port, isHttps)); c.close(); } catch (SdkException e) { throw new IOException(e); diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java new file mode 100644 index 00000000000..46207025853 --- /dev/null +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/ClientFileSystemKey.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.cloud.aws.s3; + +import java.util.Objects; + +/** + * File System Key stored in {@link FileService#fileSystems}. + * + * @author Quentin Bialota (Geomatys) + */ +final class ClientFileSystemKey { + /** + * The S3 access key. + */ + final String accessKey; + + /** + * The S3 host (if not stored on Amazon AWS Infrastructure). + */ + final String host; + + /** + * The S3 port (if not stored on Amazon AWS Infrastructure). + */ + final int port; + + /** + * Is the S3 HTTP Protocol secure (if not stored on Amazon AWS Infrastructure). + */ + final boolean isHttps; + + /** + * Creates a new file system key for the {@code FileService} with access key, host, port and protocol (secure or not secure). + * + * @param accessKey the S3 access key for this file system. + * @param host the host or {@code null} for AWS request. + * @param port the port or {@code -1} for AWS request. + * @param isHttps the protocol is secure or not + */ + public ClientFileSystemKey(String accessKey, String host, int port, boolean isHttps) { + this.accessKey = accessKey; + this.host = host; + this.port = port; + this.isHttps = isHttps; + } + + /** + * Indicates whether some other object is "equal to" this one. + * + * @param o The reference object with which to compare. + * @return {@code true} if this object is the same as the o argument; {@code false} otherwise. + */ + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof ClientFileSystemKey)) return false; + ClientFileSystemKey that = (ClientFileSystemKey) o; + return Objects.equals(accessKey, that.accessKey) && Objects.equals(host, that.host) && port == that.port && isHttps == that.isHttps; + } + + /** + * Returns a hash code value for the object. + * + * @return A hash code value for this object. + */ + @Override + public int hashCode() { + return Objects.hash(accessKey, host, port); + } +} diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java index 635dc545552..126abcb0484 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/FileService.java @@ -16,6 +16,7 @@ */ package org.apache.sis.cloud.aws.s3; +import java.net.URISyntaxException; import java.util.Map; import java.util.Set; import java.util.function.Function; @@ -65,7 +66,9 @@ * * * * "Files" are S3 keys interpreted as paths with components separated by the {@code '/'} separator. @@ -75,7 +78,8 @@ * instead of the data to access, and can be a global configuration for the server. * * @author Martin Desruisseaux (Geomatys) - * @version 1.5 + * @author Quentin Bialota (Geomatys) + * @version 1.7 * @since 1.2 */ public class FileService extends FileSystemProvider { @@ -93,6 +97,42 @@ public class FileService extends FileSystemProvider { */ private static final String DEFAULT_ACCESS_KEY = ""; + /** + * An arbitrary string used as part of the key in the {@link #fileSystems} map + * when the user did not specified explicitly a host. + * In such case, the default host is the Amazon host and is defined with the region. + * + * Host can also be defined with: + * + */ + private static final String DEFAULT_HOST_KEY = null; + + /** + * An arbitrary string used as part of the key in the {@link #fileSystems} map + * when the user did not specified explicitly a port. + * In such case, no port is assigned, the default port is used + * + * Port can also be defined with : + * + */ + private static final int DEFAULT_PORT_KEY = -1; + + /** + * A boolean used as part of the key in the {@link #fileSystems} map + * when the user did not specified explicitly a protocol. + * In such case, the default protocol is HTTPS + * + * Port can also be defined with : + * + */ + private static final boolean DEFAULT_IS_HTTPS = true; + /** * The property for the secret access key (password). * Values shall be instances of {@link String}. @@ -107,6 +147,34 @@ public class FileService extends FileSystemProvider { */ public static final String AWS_SECRET_ACCESS_KEY = "aws.secretAccessKey"; + /** + * The property for the host (mandatory if not using AWS S3). + * Values shall be instances of {@link String}. + * + * @see #newFileSystem(URI, Map) + * @since 1.7 + */ + public static final String S3_HOST_URL = "org.apache.sis.s3.hostURL"; + + /** + * The property for the port (mandatory if not using AWS S3). + * Values shall be instances of {@link Integer}. + * + * @see #newFileSystem(URI, Map) + * @since 1.7 + */ + public static final String S3_PORT = "org.apache.sis.s3.port"; + + /** + * The property for the protocol (optional). + * Values shall be instances of {@link Boolean}. + * Default value : True (HTTPS) + * + * @see #newFileSystem(URI, Map) + * @since 1.7 + */ + public static final String S3_IS_HTTPS = "org.apache.sis.s3.isHttps"; + /** * The property for the secret access key (password). * Values shall should be instances of {@link Region} or @@ -134,7 +202,7 @@ public class FileService extends FileSystemProvider { /** * All file systems created by this provider. Keys are AWS S3 access keys. */ - private final ConcurrentMap fileSystems; + private final ConcurrentMap fileSystems; /** * Creates a new provider of file systems for Amazon S3. @@ -193,7 +261,8 @@ private String getAccessKey(final URI uri) { * {@linkplain Region#of(String) convertible} to region. * * - * @param uri a URI of the form {@code "s3://accessKey@bucket/file"}. + * @param uri a URI of the form {@code "s3://accessKey@bucket/file"} + * or {@code s3://accessKey@host:port/bucket/file} (in this second case, host AND port are mandatory). * @param properties properties to configure the file system, or {@code null} if none. * @return the new file system. * @throws IllegalArgumentException if the URI or the map contains invalid values. @@ -204,6 +273,8 @@ private String getAccessKey(final URI uri) { @Override public FileSystem newFileSystem(final URI uri, final Map properties) throws IOException { final String accessKey = getAccessKey(uri); + String host = uri.getHost(); + int port = uri.getPort(); final String secret; if (accessKey == null || (secret = Containers.property(properties, AWS_SECRET_ACCESS_KEY, String.class)) == null) { throw new IllegalArgumentException(Resources.format(Resources.Keys.MissingAccessKey_2, (accessKey == null) ? 0 : 1, uri)); @@ -216,19 +287,60 @@ public FileSystem newFileSystem(final URI uri, final Map properties) t } else { region = Containers.property(properties, AWS_REGION, Region.class); } - final class Creator implements Function { + + final class Creator implements Function { /** Identifies if a new file system is created. */ boolean created; + /** Store URI Syntax exception. */ URISyntaxException exception; - /** Invoked if the map does not already contains the file system. */ - @Override public ClientFileSystem apply(final String key) { + @Override + public ClientFileSystem apply(final ClientFileSystemKey key) { created = true; - return new ClientFileSystem(FileService.this, region, key, secret, separator); + try { + return new ClientFileSystem(FileService.this, region, key.host, key.port, key.isHttps, key.accessKey, secret, separator); + } catch (URISyntaxException ex) { + created = false; + exception = ex; + return null; // Nothing added to the map + } + } + } + + Boolean isHttps; + if ((isHttps = Containers.property(properties, S3_IS_HTTPS, Boolean.class)) == null) { + isHttps = DEFAULT_IS_HTTPS; + } + + /* + * In case of Self-Hosted S3, if host and port are not found in the URI + * We check in java properties + * Else we use Default values (=> use AWS S3) + */ + if (port < 0) { + if ((host = Containers.property(properties, S3_HOST_URL, String.class)) != null) { + Integer portProp = Containers.property(properties, S3_PORT, Integer.class); + // In case of Self-Hosted S3, if port is not found in the URI, but a host is defined + if (portProp == null || portProp < 0) { + if (isHttps) { + port = 443; // Default HTTPS port + } else { + port = 80; // Default HTTP port + } + } else { + port = portProp; + } + + } else { + host = DEFAULT_HOST_KEY; + port = DEFAULT_PORT_KEY; } } + final Creator c = new Creator(); - final ClientFileSystem fs = fileSystems.computeIfAbsent(accessKey, c); + final ClientFileSystem fs = fileSystems.computeIfAbsent(new ClientFileSystemKey(accessKey, host, port, isHttps), c); if (c.created) { return fs; + } else if (c.exception != null) { + throw new IllegalArgumentException("Invalid URI: " + uri, c.exception); } throw new FileSystemAlreadyExistsException(Resources.format(Resources.Keys.FileSystemInitialized_2, 1, accessKey)); } @@ -237,28 +349,87 @@ final class Creator implements Function { * Removes the given file system from the cache. * This method is invoked after the file system has been closed. */ - final void dispose(String identifier) { + final void dispose(ClientFileSystemKey identifier) { if (identifier == null) { - identifier = DEFAULT_ACCESS_KEY; + identifier = new ClientFileSystemKey(DEFAULT_ACCESS_KEY, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS); } fileSystems.remove(identifier); } /** - * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}. + * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}, {@link #DEFAULT_HOST_KEY}, {@link #DEFAULT_PORT_KEY} and {@link #DEFAULT_IS_HTTPS}. * * @throws SdkException if the file system cannot be created. */ private ClientFileSystem getDefaultFileSystem() { - return fileSystems.computeIfAbsent(DEFAULT_ACCESS_KEY, (key) -> new ClientFileSystem(this, S3Client.create())); + return fileSystems.computeIfAbsent(new ClientFileSystemKey(DEFAULT_ACCESS_KEY, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS), (key) -> + new ClientFileSystem(this, S3Client.create(), null)); + } + + /** + * Returns the file system associated to the {@link #DEFAULT_HOST_KEY} and {@link #DEFAULT_PORT_KEY}. + * + * @param accessKey the access key + * @throws SdkException if the file system cannot be created. + */ + private ClientFileSystem getDefaultFileSystem(String accessKey) { + return fileSystems.computeIfAbsent( + new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS), + (key) -> + new ClientFileSystem(this, S3Client.create(), key.accessKey)); + } + + /** + * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}. + * + * @param host the host + * @param port the port + * @throws SdkException if the file system cannot be created. + * @throws URISyntaxException if the URI is not valid. + */ + private ClientFileSystem getDefaultFileSystem(String host, Integer port) throws URISyntaxException { + ClientFileSystemKey key = new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, DEFAULT_IS_HTTPS); + + synchronized (fileSystems) { + ClientFileSystem fs = fileSystems.get(key); + if (fs != null) return fs; + fs = new ClientFileSystem(this, null, key.host, key.port, key.isHttps, key.accessKey, null, null); + fileSystems.put(key, fs); + return fs; + } + } + + /** + * Returns the file system associated to the {@link #DEFAULT_ACCESS_KEY}. + * + * @param host the host + * @param port the port + * @param isHttps the protocol + * @throws SdkException if the file system cannot be created. + * @throws URISyntaxException if the URI is not valid. + */ + private ClientFileSystem getDefaultFileSystem(String host, Integer port, boolean isHttps) throws URISyntaxException { + ClientFileSystemKey key = new ClientFileSystemKey(DEFAULT_ACCESS_KEY, host, port, isHttps); + + // 1. Try to get the existing one first (no locking) + synchronized (fileSystems) { + ClientFileSystem fs = fileSystems.get(key); + if (fs != null) return fs; + fs = new ClientFileSystem(this, null, key.host, key.port, key.isHttps, key.accessKey, null, null); + fileSystems.put(key, fs); + return fs; + } } /** * Returns a reference to a file system that was created by the {@link #newFileSystem(URI, Map)} method. * If the file system has not been created or has been closed, * then this method throws {@link FileSystemNotFoundException}. + * If the given URI contains a port number, HTTP(S) protocol is determined by the port number (port 80 means HTTP, port 443 mean HTTPS). + * If another port is used we use the default protocol (HTTPS). + * A first attempt is made with the protocol determined from the port number, then a second attempt is made with the opposite protocol. * - * @param uri a URI of the form {@code "s3://accessKey@bucket/file"}. + * @param uri a URI of the form {@code "s3://accessKey@bucket/file"} or {@code "s3://accessKey@host:port/bucket/key"}. * @return the file system previously created by {@link #newFileSystem(URI, Map)}. * @throws IllegalArgumentException if the URI is not supported by this provider. * @throws FileSystemNotFoundException if the file system does not exist or has been closed. @@ -266,10 +437,43 @@ private ClientFileSystem getDefaultFileSystem() { @Override public FileSystem getFileSystem(final URI uri) { final String accessKey = getAccessKey(uri); - if (accessKey == null) { + final String host = uri.getHost(); + final int port = uri.getPort(); + + /* + * HTTPS or HTTP is not defined in the URI. By default, we use default value, + * except if the port is 80 or 443, in which case we use HTTP or HTTPS. + */ + boolean isHttps = DEFAULT_IS_HTTPS; + if (port == 80) { + isHttps = false; + } else if (port == 443) { + isHttps = true; + } + + if (accessKey == null && port > -1) { + // No access key, but host and port are defined => Self Hosted + try { + return getDefaultFileSystem(host, port, isHttps); + } catch (URISyntaxException ex) { + throw new IllegalArgumentException("Invalid URI: " + uri, ex); + } + } else if (accessKey == null && port < 0) { + // No access key, no host, no port => AWS S3 return getDefaultFileSystem(); + } else if (accessKey != null && port < 0) { + // Access key, no host, no port => AWS S3 + return getDefaultFileSystem(accessKey); + } + + /* + * Try to get the file system with the protocol determined from the port number, then try with the opposite protocol. + */ + ClientFileSystem fs = fileSystems.get(new ClientFileSystemKey(accessKey, host, port, isHttps)); + if (fs != null) { + return fs; } - final ClientFileSystem fs = fileSystems.get(accessKey); + fs = fileSystems.get(new ClientFileSystemKey(accessKey, host, port, !isHttps)); if (fs != null) { return fs; } @@ -277,11 +481,12 @@ public FileSystem getFileSystem(final URI uri) { } /** - * Return a {@code Path} object by converting the given {@code URI}. + * Return a {@code Path} object by converting the given {@code URI} or {@code "s3://accessKey@host:port/bucket/key"}. * The resulting {@code Path} is associated with a {@link FileSystem} * that already exists or is constructed automatically. * - * @param uri a URI of the form {@code "s3://accessKey@bucket/file"}. + * @param uri a URI of the form {@code "s3://accessKey@bucket/file"} (AWS S3) + * or {@code s3://accessKey@host:port/bucket/file} (Self-hosted S3) (in this second case, host AND port are mandatory). * @return the resulting {@code Path}. * @throws IllegalArgumentException if the URI is not supported by this provider. * @throws FileSystemNotFoundException if the file system does not exist and cannot be created automatically. @@ -289,14 +494,9 @@ public FileSystem getFileSystem(final URI uri) { @Override public Path getPath(final URI uri) { final String accessKey = getAccessKey(uri); - final ClientFileSystem fs; - if (accessKey == null) { - fs = getDefaultFileSystem(); - } else { - // TODO: we may need a way to get password here. - fs = fileSystems.computeIfAbsent(accessKey, (key) -> new ClientFileSystem(FileService.this, null, key, null, null)); - } String host = uri.getHost(); + final int port = uri.getPort(); + if (host == null) { /* * The host is null if the authority contains characters that are invalid for a host name. @@ -308,7 +508,70 @@ public Path getPath(final URI uri) { if (host == null) host = uri.toString(); throw new IllegalArgumentException(Resources.format(Resources.Keys.InvalidBucketName_1, host)); } - final String path = uri.getPath(); + + ClientFileSystem fs; + if (accessKey == null) { + if (port < 0) { + fs = getDefaultFileSystem(); + } else { + try { + fs = getDefaultFileSystem(host, port); + } catch (URISyntaxException ex) { + throw new IllegalArgumentException("Invalid URI: " + uri, ex); + } + } + } else { + // TODO: we may need a way to get password here. + // TODO: we may need a way to get SSL status here (is HTTPS or not) + ClientFileSystemKey fsKey; + + if (port < 0) { + fsKey = new ClientFileSystemKey(accessKey, DEFAULT_HOST_KEY, DEFAULT_PORT_KEY, DEFAULT_IS_HTTPS); + } else { + fsKey = new ClientFileSystemKey(accessKey, host, port, DEFAULT_IS_HTTPS); + } + + // Compute if absent + try { + synchronized (fileSystems) { + fs = fileSystems.get(fsKey); + if (fs == null) { + fs = new ClientFileSystem(this, null, fsKey.host, fsKey.port, fsKey.isHttps, fsKey.accessKey, null, null); + fileSystems.put(fsKey, fs); + } + } + } catch (URISyntaxException ex) { + throw new IllegalArgumentException("Invalid URI: " + uri, ex); + } + } + + String path = uri.getPath(); + + /* + * In case of custom host, bucket name will be the first element of the "uri.getPath()" + * We need to get the first element of this path (this will be the bucket), and the second part will be + */ + if (fs.host != null) { + if (!(fs.host.equalsIgnoreCase(DEFAULT_HOST_KEY))) { + if (path.startsWith("/")) { + path = path.substring(1); + } + String[] parts = path.split("/", 2); + if (parts.length >= 2) { + // Bucket + Path specified (=> /bucket/path/to/folder) + host = parts[0]; + path = "/" + parts[1]; + } else if (parts.length == 1) { + // Bucket specified (no path) (=> /bucket) + host = parts[0]; + path = null; + } + } + } + /* + * - "host" in this part is the S3 bucket name + * - "path" is the path in this bucket + */ return new KeyPath(fs, host, (path != null) ? new String[] {path} : CharSequences.EMPTY_ARRAY, true); } diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java index 50171eb101e..4b564ac99db 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/KeyPath.java @@ -30,6 +30,7 @@ import java.util.Iterator; import java.util.Objects; import java.util.NoSuchElementException; + import software.amazon.awssdk.services.s3.model.Bucket; import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; @@ -704,7 +705,25 @@ public URI toUri() { path = sb.toString(); } try { - return new URI(SCHEME, fs.accessKey, bucket, -1, path, null, null); + if (fs != null) { + /* + * We can address two different URI formats, depending on whether the file system is self-hosted or not: + * - Self-Hosted path : s3://accessKey@host:port/bucket/key + * - AWS path : s3://accessKey@bucket/key + * We also verify bucket presence to allow relative paths URIs. + */ + if (fs.host != null && fs.port < 0) { + throw new IllegalStateException("Self-hosted file system shall have a port number."); + } else if (fs.host == null && fs.port >= 0) { + throw new IllegalStateException("Port number specified, but no host name. Incompatible with self-hosted and AWS file system."); + } + boolean selfHosted = fs.host != null && fs.port >= 0; + String host = (selfHosted && bucket != null) ? fs.host : bucket; + int port = (selfHosted && bucket != null) ? fs.port : -1; + String uriPath = (selfHosted && bucket != null) ? "/" + bucket + (path != null ? path : "") : (path != null ? path : ""); + return new URI(SCHEME, fs.accessKey, host, port, uriPath, null, null); + } + throw new IllegalStateException("No filesystem associated with this path."); } catch (URISyntaxException e) { throw new IllegalStateException(e.getMessage(), e); } @@ -718,12 +737,25 @@ public String toString() { if (bucket == null && !isDirectory) { return key; } + /* + * We can address two different URI formats, depending on whether the file system is self-hosted or not: + * - Self-Hosted path : s3://accessKey@host:port/bucket/key + * - AWS path : s3://accessKey@bucket/key + */ + if (fs.host != null && fs.port < 0) { + throw new IllegalStateException("Self-hosted file system shall have a port number."); + } else if (fs.host == null && fs.port >= 0) { + throw new IllegalStateException("Port number specified, but no host name. Incompatible with self-hosted and AWS file system."); + } final StringBuilder sb = new StringBuilder(); if (bucket != null) { sb.append(SCHEME).append(SCHEME_SEPARATOR); if (fs.accessKey != null) { sb.append(fs.accessKey).append('@'); } + if (fs.host != null && fs.port >= 0) { + sb.append(fs.host).append(':').append(fs.port).append('/'); + } sb.append(bucket); } if (key != null) { diff --git a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/package-info.java b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/package-info.java index caf3fc4632e..695d6fe1442 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/package-info.java +++ b/endorsed/src/org.apache.sis.cloud.aws/main/org/apache/sis/cloud/aws/s3/package-info.java @@ -54,7 +54,8 @@ * All classes provided by this package are safe of usage in multi-threading environment. * * @author Martin Desruisseaux (Geomatys) - * @version 1.5 + * @author Quentin Bialota (Geomatys) + * @version 1.7 * * @see AWS SDK for Java * diff --git a/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/ClientFileSystemTest.java b/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/ClientFileSystemTest.java index bad2ad252e4..8139b8abffb 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/ClientFileSystemTest.java +++ b/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/ClientFileSystemTest.java @@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.*; import org.apache.sis.test.TestCase; +import java.net.URISyntaxException; + /** * Tests {@link ClientFileSystem}. @@ -33,18 +35,29 @@ public final class ClientFileSystemTest extends TestCase { */ private final ClientFileSystem fs; + private final ClientFileSystem fsSelfHosted; + /** * Returns the file system to use for testing purpose. */ static ClientFileSystem create() { - return new ClientFileSystem(new FileService(), null); + return new ClientFileSystem(new FileService(), null, null); + } + + /** + * Returns the file system to use for testing purpose. + * This file system is configured for self-hosted S3 server. + */ + static ClientFileSystem createSelfHosted() throws URISyntaxException { + return new ClientFileSystem(new FileService(), null, "testhost", 8581, true, null, null, null); } /** * Creates a new test case. */ - public ClientFileSystemTest() { + public ClientFileSystemTest() throws URISyntaxException { fs = create(); + fsSelfHosted = createSelfHosted(); } /** @@ -53,5 +66,6 @@ public ClientFileSystemTest() { @Test public void testGetSeparator() { assertEquals("/", fs.getSeparator()); + assertEquals("/", fsSelfHosted.getSeparator()); } } diff --git a/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/FileServiceTest.java b/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/FileServiceTest.java new file mode 100644 index 00000000000..3970a9bd271 --- /dev/null +++ b/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/FileServiceTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sis.cloud.aws.s3; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemAlreadyExistsException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +// Test dependencies +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; +import org.apache.sis.test.TestCase; + +/** + * Tests {@link FileService}. + * + * @author Quentin Bialota (Geomatys) + */ +public final class FileServiceTest extends TestCase { + + /** + * Creates a new test case. + */ + public FileServiceTest() { + } + + /** + * Tests AWS S3 FileSystem creation. + */ + @Test + public void testNewFileSystemAws() + throws URISyntaxException, IOException { + final FileService service = new FileService(); + final Map properties = new HashMap<>(); + properties.put(FileService.AWS_SECRET_ACCESS_KEY, "secret"); + + URI uri = new URI("S3://accessKey@bucket/file"); + FileSystem fs = service.newFileSystem(uri, properties); + assertNotNull(fs); + assertTrue(fs instanceof ClientFileSystem); + + assertThrows(FileSystemAlreadyExistsException.class, () -> { + service.newFileSystem(uri, properties); + }); + + // Test FileSystem fetch by URI + FileSystem fetchedFs = service.getFileSystem(uri); + assertSame(fs, fetchedFs); + } + + /** + * Tests Self-Hosted S3 FileSystem creation. + */ + @Test + public void testNewFileSystemSelfHosted() + throws URISyntaxException, IOException { + final FileService service = new FileService(); + final Map properties = new HashMap<>(); + properties.put(FileService.AWS_SECRET_ACCESS_KEY, "secret"); + + URI uri = new URI("S3://accessKey@localhost:8080/bucket/file"); + FileSystem fs = service.newFileSystem(uri, properties); + assertNotNull(fs); + assertTrue(fs instanceof ClientFileSystem); + + assertThrows(FileSystemAlreadyExistsException.class, () -> { + service.newFileSystem(uri, properties); + }); + + // Test FileSystem fetch by URI + FileSystem fetchedFs = service.getFileSystem(uri); + assertSame(fs, fetchedFs); + } + + /** + * Tests Self-Hosted S3 FileSystem creation with properties. + */ + @Test + public void testNewFileSystemSelfHostedWithPropertiesNoPort() + throws URISyntaxException, IOException { + final FileService service = new FileService(); + final Map properties = new HashMap<>(); + properties.put(FileService.AWS_SECRET_ACCESS_KEY, "secret"); + properties.put(FileService.S3_HOST_URL, "localhost"); + properties.put(FileService.S3_IS_HTTPS, false); + + URI uri = new URI("S3://accessKey@bucket/file"); + FileSystem fs = service.newFileSystem(uri, properties); + assertNotNull(fs); + assertTrue(fs instanceof ClientFileSystem); + + assertThrows(FileSystemAlreadyExistsException.class, () -> { + service.newFileSystem(uri, properties); + }); + + assertEquals(80, ((ClientFileSystem) fs).port); + Path basePath = fs.getPath("/bucket/file"); + URI generatedURI = basePath.toUri(); + assertEquals("S3://accessKey@localhost:80/bucket/file", generatedURI.toString()); + + // Test FileSystem fetch by URI + FileSystem fetchedFs = service.getFileSystem(uri); + assertNotSame(fs, fetchedFs); + fetchedFs = service.getFileSystem(generatedURI); + assertSame(fs, fetchedFs); + } + + /** + * Tests Self-Hosted S3 FileSystem creation with properties. + */ + @Test + public void testNewFileSystemSelfHostedWithProperties() + throws URISyntaxException, IOException { + final FileService service = new FileService(); + final Map properties = new HashMap<>(); + properties.put(FileService.AWS_SECRET_ACCESS_KEY, "secret"); + properties.put(FileService.S3_HOST_URL, "localhost"); + properties.put(FileService.S3_PORT, 8454); + properties.put(FileService.S3_IS_HTTPS, false); + + URI uri = new URI("S3://accessKey@bucket/file"); + FileSystem fs = service.newFileSystem(uri, properties); + assertNotNull(fs); + assertTrue(fs instanceof ClientFileSystem); + + assertThrows(FileSystemAlreadyExistsException.class, () -> { + service.newFileSystem(uri, properties); + }); + + assertEquals(8454, ((ClientFileSystem) fs).port); + Path basePath = fs.getPath("/bucket/file"); + URI generatedURI = basePath.toUri(); + assertEquals("S3://accessKey@localhost:8454/bucket/file", generatedURI.toString()); + + // Test FileSystem fetch by URI + FileSystem fetchedFs = service.getFileSystem(uri); + assertNotSame(fs, fetchedFs); + fetchedFs = service.getFileSystem(generatedURI); + assertSame(fs, fetchedFs); + } + + /** + * Tests FileSystem creation with missing secret key. + */ + @Test + public void testMissingSecretKey() + throws URISyntaxException { + final FileService service = new FileService(); + final Map properties = new HashMap<>(); + + URI uri = new URI("S3://accessKey@bucket/file"); + assertThrows(IllegalArgumentException.class, () -> { + service.newFileSystem(uri, properties); + }); + } +} diff --git a/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/KeyPathTest.java b/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/KeyPathTest.java index f3ce186b698..389b6e2bb86 100644 --- a/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/KeyPathTest.java +++ b/endorsed/src/org.apache.sis.cloud.aws/test/org/apache/sis/cloud/aws/s3/KeyPathTest.java @@ -16,6 +16,7 @@ */ package org.apache.sis.cloud.aws.s3; +import java.net.URISyntaxException; import java.nio.file.Path; import java.util.Iterator; import software.amazon.awssdk.services.s3.model.Bucket; @@ -37,19 +38,33 @@ public final class KeyPathTest extends TestCase { */ private final ClientFileSystem fs; + /** + * The file system used to test self-hosted S3 paths. + */ + private final ClientFileSystem fsSelfHosted; + /** * The path to test. */ private final KeyPath absolute, relative; + /** + * The path to test with self-hosted S3 file system. + */ + private final KeyPath absoluteSelfHosted, relativeSelfHosted; + /** * Creates a new test case. */ - public KeyPathTest() { + public KeyPathTest() throws URISyntaxException { fs = ClientFileSystemTest.create(); final KeyPath root = new KeyPath(fs, Bucket.builder().name("the-bucket").build()); absolute = new KeyPath(root, "first/second/third/the-file", false); relative = new KeyPath(fs, "second/third/the-file", false); + fsSelfHosted = ClientFileSystemTest.createSelfHosted(); + final KeyPath rootSelfHosted = new KeyPath(fsSelfHosted, Bucket.builder().name("the-bucket").build()); + absoluteSelfHosted = new KeyPath(rootSelfHosted, "first/second/third/the-file", false); + relativeSelfHosted = new KeyPath(fsSelfHosted, "second/third/the-file", false); } /** @@ -244,4 +259,26 @@ public void testCompareTo() { assertEquals(+1, relative.compareTo(absolute)); assertTrue(absolute.compareTo(new KeyPath(absolute, "first", true)) > 0); } + + /** + * Tests {@link KeyPath#toString()}. + */ + @Test + public void testToString() { + assertEquals("S3://the-bucket/first/second/third/the-file", absolute.toString()); + assertEquals("second/third/the-file", relative.toString()); + assertEquals("S3://testhost:8581/the-bucket/first/second/third/the-file", absoluteSelfHosted.toString()); + assertEquals("second/third/the-file", relativeSelfHosted.toString()); + } + + /** + * Tests {@link KeyPath#toUri()}. + */ + @Test + public void testToUri() { + assertEquals("S3://the-bucket/first/second/third/the-file", absolute.toUri().toString()); + assertEquals("S3:/second/third/the-file", relative.toUri().toString()); + assertEquals("S3://testhost:8581/the-bucket/first/second/third/the-file", absoluteSelfHosted.toUri().toString()); + assertEquals("S3:/second/third/the-file", relativeSelfHosted.toUri().toString()); + } }