✨ Add chunked uploads
This commit is contained in:
@@ -87,6 +87,7 @@ public class BucketRepo
|
||||
{
|
||||
return entries
|
||||
.filter(Files::isDirectory)
|
||||
.filter(path -> !path.getFileName().toString().startsWith("."))
|
||||
.map(this::toBucket)
|
||||
.toList();
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import jakarta.enterprise.context.ApplicationScoped;
|
||||
import jakarta.enterprise.event.Observes;
|
||||
import jakarta.inject.Inject;
|
||||
import org.eclipse.microprofile.config.inject.ConfigProperty;
|
||||
import org.jboss.logging.Logger;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
@@ -13,6 +14,8 @@ import java.util.Optional;
|
||||
@ApplicationScoped
|
||||
public class InitService
|
||||
{
|
||||
private static final Logger LOG = Logger.getLogger(InitService.class);
|
||||
|
||||
@ConfigProperty(name = "dev.dinauer.r9t.init.buckets")
|
||||
Optional<List<String>> buckets;
|
||||
|
||||
@@ -23,9 +26,14 @@ public class InitService
|
||||
{
|
||||
for (String bucket : buckets.orElse(List.of()))
|
||||
{
|
||||
if (!bucketService.exists(bucket))
|
||||
if (bucketService.exists(bucket))
|
||||
{
|
||||
LOG.infof("Bootstrap bucket '%s' already exists, skipping", bucket);
|
||||
}
|
||||
else
|
||||
{
|
||||
bucketService.create(bucket);
|
||||
LOG.infof("Bootstrapped bucket '%s'", bucket);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
package dev.dinauer.r9t.core.multipart;
|
||||
|
||||
public record CompletedPart(int partNumber, String etag)
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package dev.dinauer.r9t.core.multipart;
|
||||
|
||||
public record MultipartPart(int partNumber, String etag, long size)
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package dev.dinauer.r9t.core.multipart;
|
||||
|
||||
import java.time.Instant;
|
||||
|
||||
public record MultipartUpload(String uploadId, String bucket, String key, String contentType, Instant initiated)
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package dev.dinauer.r9t.core.multipart;
|
||||
|
||||
import dev.dinauer.r9t.core.bucket.BucketRepo;
|
||||
import dev.dinauer.r9t.core.s3.DigestCopy;
|
||||
import dev.dinauer.r9t.core.s3.FileAttributes;
|
||||
import dev.dinauer.r9t.core.s3.S3ErrorCode;
|
||||
import dev.dinauer.r9t.core.s3.S3Exception;
|
||||
import jakarta.enterprise.context.ApplicationScoped;
|
||||
import jakarta.inject.Inject;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Instant;
|
||||
import java.util.Comparator;
|
||||
import java.util.UUID;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@ApplicationScoped
|
||||
public class MultipartUploadRepo
|
||||
{
|
||||
private static final String BUCKET_ATTRIBUTE = "user.bucket";
|
||||
private static final String KEY_ATTRIBUTE = "user.key";
|
||||
private static final String INITIATED_ATTRIBUTE = "user.initiated";
|
||||
|
||||
@Inject
|
||||
BucketRepo bucketRepo;
|
||||
|
||||
public Path resolve(String uploadId)
|
||||
{
|
||||
Path root = stagingRoot();
|
||||
Path target = root.resolve(uploadId).normalize();
|
||||
if (!target.startsWith(root))
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, uploadId);
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
public boolean exists(String uploadId)
|
||||
{
|
||||
return Files.isDirectory(resolve(uploadId));
|
||||
}
|
||||
|
||||
public MultipartUpload create(String bucket, String key, String contentType)
|
||||
{
|
||||
String uploadId = UUID.randomUUID().toString().replace("-", "");
|
||||
Path dir = resolve(uploadId);
|
||||
Instant initiated = Instant.now();
|
||||
try
|
||||
{
|
||||
Files.createDirectories(dir);
|
||||
FileAttributes.write(dir, BUCKET_ATTRIBUTE, bucket);
|
||||
FileAttributes.write(dir, KEY_ATTRIBUTE, key);
|
||||
FileAttributes.write(dir, FileAttributes.CONTENT_TYPE, contentType);
|
||||
FileAttributes.write(dir, INITIATED_ATTRIBUTE, initiated.toString());
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
|
||||
}
|
||||
return new MultipartUpload(uploadId, bucket, key, contentType, initiated);
|
||||
}
|
||||
|
||||
public MultipartUpload metadata(String uploadId)
|
||||
{
|
||||
Path dir = resolve(uploadId);
|
||||
try
|
||||
{
|
||||
String bucket = FileAttributes.read(dir, BUCKET_ATTRIBUTE);
|
||||
String key = FileAttributes.read(dir, KEY_ATTRIBUTE);
|
||||
String contentType = FileAttributes.read(dir, FileAttributes.CONTENT_TYPE);
|
||||
Instant initiated = Instant.parse(FileAttributes.read(dir, INITIATED_ATTRIBUTE));
|
||||
return new MultipartUpload(uploadId, bucket, key, contentType, initiated);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
|
||||
}
|
||||
}
|
||||
|
||||
public String storePart(String uploadId, int partNumber, InputStream data)
|
||||
{
|
||||
Path partFile = partFile(uploadId, partNumber);
|
||||
try
|
||||
{
|
||||
String etag = DigestCopy.copyAndHashMd5(data, partFile);
|
||||
FileAttributes.write(partFile, FileAttributes.ETAG, etag);
|
||||
return etag;
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
|
||||
}
|
||||
}
|
||||
|
||||
public boolean partExists(String uploadId, int partNumber)
|
||||
{
|
||||
return Files.isRegularFile(partFile(uploadId, partNumber));
|
||||
}
|
||||
|
||||
public MultipartPart part(String uploadId, int partNumber)
|
||||
{
|
||||
Path partFile = partFile(uploadId, partNumber);
|
||||
try
|
||||
{
|
||||
String etag = FileAttributes.read(partFile, FileAttributes.ETAG);
|
||||
long size = Files.size(partFile);
|
||||
return new MultipartPart(partNumber, etag, size);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
|
||||
}
|
||||
}
|
||||
|
||||
public Path partFile(String uploadId, int partNumber)
|
||||
{
|
||||
return resolve(uploadId).resolve(String.format("%05d", partNumber));
|
||||
}
|
||||
|
||||
public void delete(String uploadId)
|
||||
{
|
||||
Path dir = resolve(uploadId);
|
||||
try (Stream<Path> entries = Files.walk(dir))
|
||||
{
|
||||
for (Path path : entries.sorted(Comparator.reverseOrder()).toList())
|
||||
{
|
||||
Files.deleteIfExists(path);
|
||||
}
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
|
||||
}
|
||||
}
|
||||
|
||||
private Path stagingRoot()
|
||||
{
|
||||
Path root = bucketRepo.root().resolve(".multipart");
|
||||
try
|
||||
{
|
||||
Files.createDirectories(root);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, root.toString(), e);
|
||||
}
|
||||
return root;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package dev.dinauer.r9t.core.multipart;
|
||||
|
||||
import dev.dinauer.r9t.core.bucket.BucketService;
|
||||
import dev.dinauer.r9t.core.object.ObjectRepo;
|
||||
import dev.dinauer.r9t.core.object.S3Object;
|
||||
import dev.dinauer.r9t.core.s3.DigestCopy;
|
||||
import dev.dinauer.r9t.core.s3.S3ErrorCode;
|
||||
import dev.dinauer.r9t.core.s3.S3Exception;
|
||||
import jakarta.enterprise.context.ApplicationScoped;
|
||||
import jakarta.inject.Inject;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.SequenceInputStream;
|
||||
import java.io.UncheckedIOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import java.util.Vector;
|
||||
|
||||
@ApplicationScoped
|
||||
public class MultipartUploadService
|
||||
{
|
||||
@Inject
|
||||
MultipartUploadRepo multipartUploadRepo;
|
||||
|
||||
@Inject
|
||||
ObjectRepo objectRepo;
|
||||
|
||||
@Inject
|
||||
BucketService bucketService;
|
||||
|
||||
public MultipartUpload initiate(String bucket, String key, String contentType)
|
||||
{
|
||||
bucketService.require(bucket);
|
||||
String resolvedContentType = contentType;
|
||||
if (resolvedContentType == null)
|
||||
{
|
||||
resolvedContentType = "application/octet-stream";
|
||||
}
|
||||
return multipartUploadRepo.create(bucket, key, resolvedContentType);
|
||||
}
|
||||
|
||||
public String uploadPart(String bucket, String key, String uploadId, Integer partNumber, InputStream data)
|
||||
{
|
||||
require(bucket, key, uploadId);
|
||||
if (partNumber == null || partNumber < 1 || partNumber > 10000)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, "partNumber");
|
||||
}
|
||||
return multipartUploadRepo.storePart(uploadId, partNumber, data);
|
||||
}
|
||||
|
||||
public S3Object complete(String bucket, String key, String uploadId, List<CompletedPart> parts)
|
||||
{
|
||||
MultipartUpload upload = require(bucket, key, uploadId);
|
||||
if (parts.isEmpty())
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, uploadId);
|
||||
}
|
||||
List<MultipartPart> resolvedParts = resolveParts(uploadId, parts);
|
||||
String etag = combineEtags(resolvedParts);
|
||||
try (InputStream assembled = assemble(uploadId, resolvedParts))
|
||||
{
|
||||
objectRepo.storeWithEtag(bucket, key, upload.contentType(), assembled, etag);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
|
||||
}
|
||||
multipartUploadRepo.delete(uploadId);
|
||||
return objectRepo.metadata(bucket, key);
|
||||
}
|
||||
|
||||
public void abort(String bucket, String key, String uploadId)
|
||||
{
|
||||
require(bucket, key, uploadId);
|
||||
multipartUploadRepo.delete(uploadId);
|
||||
}
|
||||
|
||||
private MultipartUpload require(String bucket, String key, String uploadId)
|
||||
{
|
||||
bucketService.require(bucket);
|
||||
if (!multipartUploadRepo.exists(uploadId))
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD, uploadId);
|
||||
}
|
||||
MultipartUpload upload = multipartUploadRepo.metadata(uploadId);
|
||||
if (!upload.bucket().equals(bucket) || !upload.key().equals(key))
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD, uploadId);
|
||||
}
|
||||
return upload;
|
||||
}
|
||||
|
||||
private List<MultipartPart> resolveParts(String uploadId, List<CompletedPart> parts)
|
||||
{
|
||||
List<MultipartPart> resolved = new ArrayList<>();
|
||||
int previousPartNumber = 0;
|
||||
for (CompletedPart requested : parts)
|
||||
{
|
||||
if (requested.partNumber() <= previousPartNumber)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId);
|
||||
}
|
||||
if (!multipartUploadRepo.partExists(uploadId, requested.partNumber()))
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId);
|
||||
}
|
||||
MultipartPart stored = multipartUploadRepo.part(uploadId, requested.partNumber());
|
||||
if (!stored.etag().equalsIgnoreCase(requested.etag()))
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId);
|
||||
}
|
||||
resolved.add(stored);
|
||||
previousPartNumber = requested.partNumber();
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
private String combineEtags(List<MultipartPart> parts)
|
||||
{
|
||||
MessageDigest digest = DigestCopy.newMd5();
|
||||
for (MultipartPart part : parts)
|
||||
{
|
||||
digest.update(HexFormat.of().parseHex(part.etag()));
|
||||
}
|
||||
return HexFormat.of().formatHex(digest.digest()) + "-" + parts.size();
|
||||
}
|
||||
|
||||
private InputStream assemble(String uploadId, List<MultipartPart> parts)
|
||||
{
|
||||
Vector<InputStream> streams = new Vector<>();
|
||||
for (MultipartPart part : parts)
|
||||
{
|
||||
Path partFile = multipartUploadRepo.partFile(uploadId, part.partNumber());
|
||||
try
|
||||
{
|
||||
streams.add(Files.newInputStream(partFile));
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
}
|
||||
return new SequenceInputStream(Collections.enumeration(streams));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package dev.dinauer.r9t.core.multipart;
|
||||
|
||||
import dev.dinauer.r9t.core.object.S3Object;
|
||||
import dev.dinauer.r9t.core.s3.S3ErrorCode;
|
||||
import dev.dinauer.r9t.core.s3.S3Exception;
|
||||
import dev.dinauer.r9t.core.s3.S3Xml;
|
||||
import jakarta.enterprise.context.ApplicationScoped;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.w3c.dom.Document;
|
||||
import org.w3c.dom.Element;
|
||||
import org.w3c.dom.NodeList;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import javax.xml.parsers.DocumentBuilder;
|
||||
import javax.xml.parsers.DocumentBuilderFactory;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@ApplicationScoped
|
||||
public class MultipartUploadXmlMapper
|
||||
{
|
||||
public String map(MultipartUpload upload)
|
||||
{
|
||||
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
|
||||
+ "<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n"
|
||||
+ " <Bucket>" + S3Xml.escape(upload.bucket()) + "</Bucket>\n"
|
||||
+ " <Key>" + S3Xml.escape(upload.key()) + "</Key>\n"
|
||||
+ " <UploadId>" + S3Xml.escape(upload.uploadId()) + "</UploadId>\n"
|
||||
+ "</InitiateMultipartUploadResult>";
|
||||
}
|
||||
|
||||
public String map(String bucket, S3Object completed)
|
||||
{
|
||||
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
|
||||
+ "<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n"
|
||||
+ " <Bucket>" + S3Xml.escape(bucket) + "</Bucket>\n"
|
||||
+ " <Key>" + S3Xml.escape(completed.key()) + "</Key>\n"
|
||||
+ " <ETag>"" + completed.etag() + ""</ETag>\n"
|
||||
+ "</CompleteMultipartUploadResult>";
|
||||
}
|
||||
|
||||
public List<CompletedPart> map(InputStream body)
|
||||
{
|
||||
try
|
||||
{
|
||||
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
|
||||
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
|
||||
DocumentBuilder builder = factory.newDocumentBuilder();
|
||||
Document document = builder.parse(body);
|
||||
NodeList partNodes = document.getElementsByTagName("Part");
|
||||
List<CompletedPart> parts = new ArrayList<>();
|
||||
for (int i = 0; i < partNodes.getLength(); i++)
|
||||
{
|
||||
parts.add(toCompletedPart((Element) partNodes.item(i)));
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
catch (ParserConfigurationException | SAXException | IOException | NumberFormatException | NullPointerException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.MALFORMED_XML, "CompleteMultipartUpload", e);
|
||||
}
|
||||
}
|
||||
|
||||
private CompletedPart toCompletedPart(Element partElement)
|
||||
{
|
||||
int partNumber = Integer.parseInt(text(partElement, "PartNumber"));
|
||||
String etag = StringUtils.strip(text(partElement, "ETag"), "\"");
|
||||
return new CompletedPart(partNumber, etag);
|
||||
}
|
||||
|
||||
private String text(Element parent, String tagName)
|
||||
{
|
||||
return parent.getElementsByTagName(tagName).item(0).getTextContent();
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package dev.dinauer.r9t.core.object;
|
||||
|
||||
import dev.dinauer.r9t.core.bucket.BucketRepo;
|
||||
import dev.dinauer.r9t.core.s3.DigestCopy;
|
||||
import dev.dinauer.r9t.core.s3.FileAttributes;
|
||||
import dev.dinauer.r9t.core.s3.S3ErrorCode;
|
||||
import dev.dinauer.r9t.core.s3.S3Exception;
|
||||
import jakarta.enterprise.context.ApplicationScoped;
|
||||
@@ -8,26 +10,16 @@ import jakarta.inject.Inject;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.nio.file.attribute.UserDefinedFileAttributeView;
|
||||
import java.security.DigestInputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.HexFormat;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
@ApplicationScoped
|
||||
public class ObjectRepo
|
||||
{
|
||||
private static final String ETAG_ATTRIBUTE = "user.etag";
|
||||
private static final String CONTENT_TYPE_ATTRIBUTE = "user.contentType";
|
||||
|
||||
@Inject
|
||||
BucketRepo bucketRepo;
|
||||
|
||||
@@ -53,17 +45,28 @@ public class ObjectRepo
|
||||
try
|
||||
{
|
||||
Files.createDirectories(target.getParent());
|
||||
MessageDigest digest = MessageDigest.getInstance("MD5");
|
||||
try (DigestInputStream digestInputStream = new DigestInputStream(data, digest))
|
||||
{
|
||||
Files.copy(digestInputStream, target, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
String etag = HexFormat.of().formatHex(digest.digest());
|
||||
writeAttribute(target, ETAG_ATTRIBUTE, etag);
|
||||
writeAttribute(target, CONTENT_TYPE_ATTRIBUTE, contentType);
|
||||
String etag = DigestCopy.copyAndHashMd5(data, target);
|
||||
FileAttributes.write(target, FileAttributes.ETAG, etag);
|
||||
FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType);
|
||||
return etag;
|
||||
}
|
||||
catch (IOException | NoSuchAlgorithmException e)
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e);
|
||||
}
|
||||
}
|
||||
|
||||
public void storeWithEtag(String bucket, String key, String contentType, InputStream data, String etag)
|
||||
{
|
||||
Path target = resolve(bucket, key);
|
||||
try
|
||||
{
|
||||
Files.createDirectories(target.getParent());
|
||||
Files.copy(data, target, StandardCopyOption.REPLACE_EXISTING);
|
||||
FileAttributes.write(target, FileAttributes.ETAG, etag);
|
||||
FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType);
|
||||
}
|
||||
catch (IOException e)
|
||||
{
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e);
|
||||
}
|
||||
@@ -75,8 +78,8 @@ public class ObjectRepo
|
||||
try
|
||||
{
|
||||
BasicFileAttributes attributes = Files.readAttributes(target, BasicFileAttributes.class);
|
||||
String etag = readAttribute(target, ETAG_ATTRIBUTE);
|
||||
String contentType = readAttribute(target, CONTENT_TYPE_ATTRIBUTE);
|
||||
String etag = FileAttributes.read(target, FileAttributes.ETAG);
|
||||
String contentType = FileAttributes.read(target, FileAttributes.CONTENT_TYPE);
|
||||
return new S3Object(key, attributes.size(), attributes.lastModifiedTime().toInstant(), etag, contentType);
|
||||
}
|
||||
catch (IOException e)
|
||||
@@ -117,19 +120,4 @@ public class ObjectRepo
|
||||
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeAttribute(Path file, String name, String value) throws IOException
|
||||
{
|
||||
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
|
||||
view.write(name, StandardCharsets.UTF_8.encode(value));
|
||||
}
|
||||
|
||||
private String readAttribute(Path file, String name) throws IOException
|
||||
{
|
||||
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
|
||||
ByteBuffer buffer = ByteBuffer.allocate(view.size(name));
|
||||
view.read(name, buffer);
|
||||
buffer.flip();
|
||||
return StandardCharsets.UTF_8.decode(buffer).toString();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package dev.dinauer.r9t.core.object;
|
||||
|
||||
import dev.dinauer.r9t.core.auth.AwsChunkedInputStream;
|
||||
import dev.dinauer.r9t.core.multipart.CompletedPart;
|
||||
import dev.dinauer.r9t.core.multipart.MultipartUpload;
|
||||
import dev.dinauer.r9t.core.multipart.MultipartUploadService;
|
||||
import dev.dinauer.r9t.core.multipart.MultipartUploadXmlMapper;
|
||||
import io.quarkus.security.Authenticated;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import jakarta.inject.Inject;
|
||||
@@ -9,15 +13,20 @@ import jakarta.ws.rs.DELETE;
|
||||
import jakarta.ws.rs.GET;
|
||||
import jakarta.ws.rs.HEAD;
|
||||
import jakarta.ws.rs.HeaderParam;
|
||||
import jakarta.ws.rs.POST;
|
||||
import jakarta.ws.rs.PUT;
|
||||
import jakarta.ws.rs.Path;
|
||||
import jakarta.ws.rs.PathParam;
|
||||
import jakarta.ws.rs.Produces;
|
||||
import jakarta.ws.rs.QueryParam;
|
||||
import jakarta.ws.rs.core.Context;
|
||||
import jakarta.ws.rs.core.MediaType;
|
||||
import jakarta.ws.rs.core.Response;
|
||||
import jakarta.ws.rs.core.UriInfo;
|
||||
|
||||
import java.io.InputStream;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
@Path("/{bucket}")
|
||||
@Authenticated
|
||||
@@ -29,6 +38,12 @@ public class ObjectResource
|
||||
@Inject
|
||||
ObjectXmlMapper objectXmlMapper;
|
||||
|
||||
@Inject
|
||||
MultipartUploadService multipartUploadService;
|
||||
|
||||
@Inject
|
||||
MultipartUploadXmlMapper multipartUploadXmlMapper;
|
||||
|
||||
@GET
|
||||
@Produces(MediaType.APPLICATION_XML)
|
||||
public Response list(@PathParam("bucket") String bucket)
|
||||
@@ -40,6 +55,7 @@ public class ObjectResource
|
||||
@Path("{key:.+}")
|
||||
@Consumes(MediaType.WILDCARD)
|
||||
public Response put(@PathParam("bucket") String bucket, @PathParam("key") String key,
|
||||
@QueryParam("uploadId") String uploadId, @QueryParam("partNumber") Integer partNumber,
|
||||
@HeaderParam("Content-Type") String contentType,
|
||||
@HeaderParam("X-Amz-Content-Sha256") String contentSha256, InputStream body)
|
||||
{
|
||||
@@ -48,10 +64,33 @@ public class ObjectResource
|
||||
{
|
||||
payload = new AwsChunkedInputStream(body);
|
||||
}
|
||||
if (uploadId != null)
|
||||
{
|
||||
String partEtag = multipartUploadService.uploadPart(bucket, key, uploadId, partNumber, payload);
|
||||
return Response.ok().header("ETag", "\"" + partEtag + "\"").build();
|
||||
}
|
||||
String etag = objectService.put(bucket, key, contentType, payload);
|
||||
return Response.ok().header("ETag", "\"" + etag + "\"").build();
|
||||
}
|
||||
|
||||
@POST
|
||||
@Path("{key:.+}")
|
||||
@Consumes(MediaType.WILDCARD)
|
||||
@Produces(MediaType.APPLICATION_XML)
|
||||
public Response post(@PathParam("bucket") String bucket, @PathParam("key") String key,
|
||||
@Context UriInfo uriInfo, @QueryParam("uploadId") String uploadId,
|
||||
@HeaderParam("Content-Type") String contentType, InputStream body)
|
||||
{
|
||||
if (uriInfo.getQueryParameters().containsKey("uploads"))
|
||||
{
|
||||
MultipartUpload upload = multipartUploadService.initiate(bucket, key, contentType);
|
||||
return Response.ok(multipartUploadXmlMapper.map(upload)).build();
|
||||
}
|
||||
List<CompletedPart> parts = multipartUploadXmlMapper.map(body);
|
||||
S3Object completed = multipartUploadService.complete(bucket, key, uploadId, parts);
|
||||
return Response.ok(multipartUploadXmlMapper.map(bucket, completed)).build();
|
||||
}
|
||||
|
||||
@GET
|
||||
@Path("{key:.+}")
|
||||
public Response get(@PathParam("bucket") String bucket, @PathParam("key") String key)
|
||||
@@ -79,8 +118,14 @@ public class ObjectResource
|
||||
|
||||
@DELETE
|
||||
@Path("{key:.+}")
|
||||
public Response delete(@PathParam("bucket") String bucket, @PathParam("key") String key)
|
||||
public Response delete(@PathParam("bucket") String bucket, @PathParam("key") String key,
|
||||
@QueryParam("uploadId") String uploadId)
|
||||
{
|
||||
if (uploadId != null)
|
||||
{
|
||||
multipartUploadService.abort(bucket, key, uploadId);
|
||||
return Response.noContent().build();
|
||||
}
|
||||
objectService.delete(bucket, key);
|
||||
return Response.noContent().build();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package dev.dinauer.r9t.core.s3;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.security.DigestInputStream;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.HexFormat;
|
||||
|
||||
public final class DigestCopy
|
||||
{
|
||||
private DigestCopy()
|
||||
{
|
||||
}
|
||||
|
||||
public static String copyAndHashMd5(InputStream data, Path target) throws IOException
|
||||
{
|
||||
MessageDigest digest = newMd5();
|
||||
try (DigestInputStream digestInputStream = new DigestInputStream(data, digest))
|
||||
{
|
||||
Files.copy(digestInputStream, target, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
return HexFormat.of().formatHex(digest.digest());
|
||||
}
|
||||
|
||||
public static MessageDigest newMd5()
|
||||
{
|
||||
try
|
||||
{
|
||||
return MessageDigest.getInstance("MD5");
|
||||
}
|
||||
catch (NoSuchAlgorithmException e)
|
||||
{
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package dev.dinauer.r9t.core.s3;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.attribute.UserDefinedFileAttributeView;
|
||||
|
||||
public final class FileAttributes
|
||||
{
|
||||
public static final String ETAG = "user.etag";
|
||||
public static final String CONTENT_TYPE = "user.contentType";
|
||||
|
||||
private FileAttributes()
|
||||
{
|
||||
}
|
||||
|
||||
public static void write(Path file, String name, String value) throws IOException
|
||||
{
|
||||
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
|
||||
view.write(name, StandardCharsets.UTF_8.encode(value));
|
||||
}
|
||||
|
||||
public static String read(Path file, String name) throws IOException
|
||||
{
|
||||
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
|
||||
ByteBuffer buffer = ByteBuffer.allocate(view.size(name));
|
||||
view.read(name, buffer);
|
||||
buffer.flip();
|
||||
return StandardCharsets.UTF_8.decode(buffer).toString();
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,9 @@ public enum S3ErrorCode
|
||||
BUCKET_NOT_EMPTY("BucketNotEmpty", 409, "The bucket you tried to delete is not empty"),
|
||||
INVALID_BUCKET_NAME("InvalidBucketName", 400, "The specified bucket is not valid"),
|
||||
INVALID_ARGUMENT("InvalidArgument", 400, "Invalid argument"),
|
||||
NO_SUCH_UPLOAD("NoSuchUpload", 404, "The specified multipart upload does not exist"),
|
||||
INVALID_PART("InvalidPart", 400, "One or more of the specified parts could not be found"),
|
||||
MALFORMED_XML("MalformedXML", 400, "The XML you provided was not well-formed or did not validate against our published schema"),
|
||||
INTERNAL_ERROR("InternalError", 500, "We encountered an internal error");
|
||||
|
||||
private final String code;
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
%dev.dev.dinauer.r9t.credentials.access-key=devkey
|
||||
%dev.dev.dinauer.r9t.credentials.secret-key=devsecret
|
||||
quarkus.http.limits.max-body-size=5G
|
||||
@@ -0,0 +1,142 @@
|
||||
package dev.dinauer.r9t.core.multipart;
|
||||
|
||||
import dev.dinauer.r9t.core.auth.SigV4TestSigner;
|
||||
import io.quarkus.test.common.http.TestHTTPResource;
|
||||
import io.quarkus.test.junit.QuarkusTest;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
@QuarkusTest
|
||||
public class MultipartUploadTest
|
||||
{
|
||||
private static final String BUCKET = "multipart-test-bucket";
|
||||
private static final String KEY = "multipart-test.txt";
|
||||
private static final String ACCESS_KEY = "testkey";
|
||||
private static final String SECRET_KEY = "testsecret";
|
||||
|
||||
@TestHTTPResource
|
||||
URI baseUri;
|
||||
|
||||
private final HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build();
|
||||
|
||||
@Test
|
||||
public void completeUploadAssemblesObject() throws Exception
|
||||
{
|
||||
createBucket();
|
||||
|
||||
HttpResponse<String> initiate = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploads="),
|
||||
new byte[0], "text/plain");
|
||||
assertEquals(200, initiate.statusCode());
|
||||
String uploadId = extract(initiate.body(), "UploadId");
|
||||
|
||||
String etag1 = uploadPart(uploadId, 1, "hello ");
|
||||
String etag2 = uploadPart(uploadId, 2, "world");
|
||||
|
||||
String completeBody = "<CompleteMultipartUpload>"
|
||||
+ "<Part><PartNumber>1</PartNumber><ETag>\"" + etag1 + "\"</ETag></Part>"
|
||||
+ "<Part><PartNumber>2</PartNumber><ETag>\"" + etag2 + "\"</ETag></Part>"
|
||||
+ "</CompleteMultipartUpload>";
|
||||
HttpResponse<String> complete = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploadId=" + uploadId),
|
||||
completeBody.getBytes(StandardCharsets.UTF_8), null);
|
||||
assertEquals(200, complete.statusCode());
|
||||
String finalEtag = extract(complete.body(), "ETag");
|
||||
assertTrue(Pattern.matches("[0-9a-f]{32}-2", finalEtag), "unexpected ETag: " + finalEtag);
|
||||
|
||||
HttpResponse<byte[]> get = sendForBytes("GET", baseUri.resolve(BUCKET + "/" + KEY));
|
||||
assertEquals(200, get.statusCode());
|
||||
assertEquals("text/plain", get.headers().firstValue("Content-Type").orElse(""));
|
||||
assertEquals("hello world", new String(get.body(), StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void abortDiscardsStagedParts() throws Exception
|
||||
{
|
||||
createBucket();
|
||||
|
||||
HttpResponse<String> initiate = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploads="),
|
||||
new byte[0], "text/plain");
|
||||
String uploadId = extract(initiate.body(), "UploadId");
|
||||
|
||||
uploadPart(uploadId, 1, "hello ");
|
||||
|
||||
HttpResponse<Void> abort = send("DELETE", baseUri.resolve(BUCKET + "/" + KEY + "?uploadId=" + uploadId),
|
||||
new byte[0], null);
|
||||
assertEquals(204, abort.statusCode());
|
||||
|
||||
HttpResponse<Void> uploadAfterAbort = send("PUT",
|
||||
baseUri.resolve(BUCKET + "/" + KEY + "?partNumber=2&uploadId=" + uploadId), "world".getBytes(StandardCharsets.UTF_8), null);
|
||||
assertEquals(404, uploadAfterAbort.statusCode());
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
public void cleanup() throws Exception
|
||||
{
|
||||
send("DELETE", baseUri.resolve(BUCKET + "/" + KEY), new byte[0], null);
|
||||
send("DELETE", baseUri.resolve(BUCKET), new byte[0], null);
|
||||
}
|
||||
|
||||
private void createBucket() throws Exception
|
||||
{
|
||||
HttpResponse<Void> createBucket = send("PUT", baseUri.resolve(BUCKET), new byte[0], null);
|
||||
assertEquals(200, createBucket.statusCode());
|
||||
}
|
||||
|
||||
private String uploadPart(String uploadId, int partNumber, String content) throws Exception
|
||||
{
|
||||
byte[] body = content.getBytes(StandardCharsets.UTF_8);
|
||||
HttpResponse<Void> put = send("PUT",
|
||||
baseUri.resolve(BUCKET + "/" + KEY + "?partNumber=" + partNumber + "&uploadId=" + uploadId), body, null);
|
||||
assertEquals(200, put.statusCode());
|
||||
return put.headers().firstValue("ETag").orElseThrow().replace("\"", "");
|
||||
}
|
||||
|
||||
private String extract(String xml, String tag)
|
||||
{
|
||||
Matcher matcher = Pattern.compile("<" + tag + ">(.*?)</" + tag + ">").matcher(xml);
|
||||
assertTrue(matcher.find(), "missing <" + tag + "> in: " + xml);
|
||||
return matcher.group(1).replace(""", "").replace("\"", "");
|
||||
}
|
||||
|
||||
private HttpResponse<Void> send(String method, URI uri, byte[] body, String contentType) throws Exception
|
||||
{
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
|
||||
.method(method, HttpRequest.BodyPublishers.ofByteArray(body));
|
||||
if (contentType != null)
|
||||
{
|
||||
builder.header("Content-Type", contentType);
|
||||
}
|
||||
SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body);
|
||||
return client.send(builder.build(), HttpResponse.BodyHandlers.discarding());
|
||||
}
|
||||
|
||||
private HttpResponse<String> sendForString(String method, URI uri, byte[] body, String contentType) throws Exception
|
||||
{
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
|
||||
.method(method, HttpRequest.BodyPublishers.ofByteArray(body));
|
||||
if (contentType != null)
|
||||
{
|
||||
builder.header("Content-Type", contentType);
|
||||
}
|
||||
SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body);
|
||||
return client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
|
||||
}
|
||||
|
||||
private HttpResponse<byte[]> sendForBytes(String method, URI uri) throws Exception
|
||||
{
|
||||
HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
|
||||
.method(method, HttpRequest.BodyPublishers.ofByteArray(new byte[0]));
|
||||
SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, new byte[0]);
|
||||
return client.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user