✨ Add chunked uploads

This commit is contained in:
2026-08-21 15:14:32 +02:00
parent 9d90b901e1
commit 0b8e36fe58
15 changed files with 698 additions and 40 deletions
@@ -87,6 +87,7 @@ public class BucketRepo
{ {
return entries return entries
.filter(Files::isDirectory) .filter(Files::isDirectory)
.filter(path -> !path.getFileName().toString().startsWith("."))
.map(this::toBucket) .map(this::toBucket)
.toList(); .toList();
} }
@@ -6,6 +6,7 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes; import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject; import jakarta.inject.Inject;
import org.eclipse.microprofile.config.inject.ConfigProperty; import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.jboss.logging.Logger;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
@@ -13,6 +14,8 @@ import java.util.Optional;
@ApplicationScoped @ApplicationScoped
public class InitService public class InitService
{ {
private static final Logger LOG = Logger.getLogger(InitService.class);
@ConfigProperty(name = "dev.dinauer.r9t.init.buckets") @ConfigProperty(name = "dev.dinauer.r9t.init.buckets")
Optional<List<String>> buckets; Optional<List<String>> buckets;
@@ -23,9 +26,14 @@ public class InitService
{ {
for (String bucket : buckets.orElse(List.of())) for (String bucket : buckets.orElse(List.of()))
{ {
if (!bucketService.exists(bucket)) if (bucketService.exists(bucket))
{
LOG.infof("Bootstrap bucket '%s' already exists, skipping", bucket);
}
else
{ {
bucketService.create(bucket); bucketService.create(bucket);
LOG.infof("Bootstrapped bucket '%s'", bucket);
} }
} }
} }
@@ -0,0 +1,5 @@
package dev.dinauer.r9t.core.multipart;
public record CompletedPart(int partNumber, String etag)
{
}
@@ -0,0 +1,5 @@
package dev.dinauer.r9t.core.multipart;
public record MultipartPart(int partNumber, String etag, long size)
{
}
@@ -0,0 +1,7 @@
package dev.dinauer.r9t.core.multipart;
import java.time.Instant;
public record MultipartUpload(String uploadId, String bucket, String key, String contentType, Instant initiated)
{
}
@@ -0,0 +1,152 @@
package dev.dinauer.r9t.core.multipart;
import dev.dinauer.r9t.core.bucket.BucketRepo;
import dev.dinauer.r9t.core.s3.DigestCopy;
import dev.dinauer.r9t.core.s3.FileAttributes;
import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Instant;
import java.util.Comparator;
import java.util.UUID;
import java.util.stream.Stream;
@ApplicationScoped
public class MultipartUploadRepo
{
private static final String BUCKET_ATTRIBUTE = "user.bucket";
private static final String KEY_ATTRIBUTE = "user.key";
private static final String INITIATED_ATTRIBUTE = "user.initiated";
@Inject
BucketRepo bucketRepo;
public Path resolve(String uploadId)
{
Path root = stagingRoot();
Path target = root.resolve(uploadId).normalize();
if (!target.startsWith(root))
{
throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, uploadId);
}
return target;
}
public boolean exists(String uploadId)
{
return Files.isDirectory(resolve(uploadId));
}
public MultipartUpload create(String bucket, String key, String contentType)
{
String uploadId = UUID.randomUUID().toString().replace("-", "");
Path dir = resolve(uploadId);
Instant initiated = Instant.now();
try
{
Files.createDirectories(dir);
FileAttributes.write(dir, BUCKET_ATTRIBUTE, bucket);
FileAttributes.write(dir, KEY_ATTRIBUTE, key);
FileAttributes.write(dir, FileAttributes.CONTENT_TYPE, contentType);
FileAttributes.write(dir, INITIATED_ATTRIBUTE, initiated.toString());
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
}
return new MultipartUpload(uploadId, bucket, key, contentType, initiated);
}
public MultipartUpload metadata(String uploadId)
{
Path dir = resolve(uploadId);
try
{
String bucket = FileAttributes.read(dir, BUCKET_ATTRIBUTE);
String key = FileAttributes.read(dir, KEY_ATTRIBUTE);
String contentType = FileAttributes.read(dir, FileAttributes.CONTENT_TYPE);
Instant initiated = Instant.parse(FileAttributes.read(dir, INITIATED_ATTRIBUTE));
return new MultipartUpload(uploadId, bucket, key, contentType, initiated);
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
}
}
public String storePart(String uploadId, int partNumber, InputStream data)
{
Path partFile = partFile(uploadId, partNumber);
try
{
String etag = DigestCopy.copyAndHashMd5(data, partFile);
FileAttributes.write(partFile, FileAttributes.ETAG, etag);
return etag;
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
}
}
public boolean partExists(String uploadId, int partNumber)
{
return Files.isRegularFile(partFile(uploadId, partNumber));
}
public MultipartPart part(String uploadId, int partNumber)
{
Path partFile = partFile(uploadId, partNumber);
try
{
String etag = FileAttributes.read(partFile, FileAttributes.ETAG);
long size = Files.size(partFile);
return new MultipartPart(partNumber, etag, size);
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
}
}
public Path partFile(String uploadId, int partNumber)
{
return resolve(uploadId).resolve(String.format("%05d", partNumber));
}
public void delete(String uploadId)
{
Path dir = resolve(uploadId);
try (Stream<Path> entries = Files.walk(dir))
{
for (Path path : entries.sorted(Comparator.reverseOrder()).toList())
{
Files.deleteIfExists(path);
}
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
}
}
private Path stagingRoot()
{
Path root = bucketRepo.root().resolve(".multipart");
try
{
Files.createDirectories(root);
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, root.toString(), e);
}
return root;
}
}
@@ -0,0 +1,152 @@
package dev.dinauer.r9t.core.multipart;
import dev.dinauer.r9t.core.bucket.BucketService;
import dev.dinauer.r9t.core.object.ObjectRepo;
import dev.dinauer.r9t.core.object.S3Object;
import dev.dinauer.r9t.core.s3.DigestCopy;
import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import java.io.IOException;
import java.io.InputStream;
import java.io.SequenceInputStream;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HexFormat;
import java.util.List;
import java.util.Vector;
@ApplicationScoped
public class MultipartUploadService
{
@Inject
MultipartUploadRepo multipartUploadRepo;
@Inject
ObjectRepo objectRepo;
@Inject
BucketService bucketService;
public MultipartUpload initiate(String bucket, String key, String contentType)
{
bucketService.require(bucket);
String resolvedContentType = contentType;
if (resolvedContentType == null)
{
resolvedContentType = "application/octet-stream";
}
return multipartUploadRepo.create(bucket, key, resolvedContentType);
}
public String uploadPart(String bucket, String key, String uploadId, Integer partNumber, InputStream data)
{
require(bucket, key, uploadId);
if (partNumber == null || partNumber < 1 || partNumber > 10000)
{
throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, "partNumber");
}
return multipartUploadRepo.storePart(uploadId, partNumber, data);
}
public S3Object complete(String bucket, String key, String uploadId, List<CompletedPart> parts)
{
MultipartUpload upload = require(bucket, key, uploadId);
if (parts.isEmpty())
{
throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, uploadId);
}
List<MultipartPart> resolvedParts = resolveParts(uploadId, parts);
String etag = combineEtags(resolvedParts);
try (InputStream assembled = assemble(uploadId, resolvedParts))
{
objectRepo.storeWithEtag(bucket, key, upload.contentType(), assembled, etag);
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
}
multipartUploadRepo.delete(uploadId);
return objectRepo.metadata(bucket, key);
}
public void abort(String bucket, String key, String uploadId)
{
require(bucket, key, uploadId);
multipartUploadRepo.delete(uploadId);
}
private MultipartUpload require(String bucket, String key, String uploadId)
{
bucketService.require(bucket);
if (!multipartUploadRepo.exists(uploadId))
{
throw new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD, uploadId);
}
MultipartUpload upload = multipartUploadRepo.metadata(uploadId);
if (!upload.bucket().equals(bucket) || !upload.key().equals(key))
{
throw new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD, uploadId);
}
return upload;
}
private List<MultipartPart> resolveParts(String uploadId, List<CompletedPart> parts)
{
List<MultipartPart> resolved = new ArrayList<>();
int previousPartNumber = 0;
for (CompletedPart requested : parts)
{
if (requested.partNumber() <= previousPartNumber)
{
throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId);
}
if (!multipartUploadRepo.partExists(uploadId, requested.partNumber()))
{
throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId);
}
MultipartPart stored = multipartUploadRepo.part(uploadId, requested.partNumber());
if (!stored.etag().equalsIgnoreCase(requested.etag()))
{
throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId);
}
resolved.add(stored);
previousPartNumber = requested.partNumber();
}
return resolved;
}
private String combineEtags(List<MultipartPart> parts)
{
MessageDigest digest = DigestCopy.newMd5();
for (MultipartPart part : parts)
{
digest.update(HexFormat.of().parseHex(part.etag()));
}
return HexFormat.of().formatHex(digest.digest()) + "-" + parts.size();
}
private InputStream assemble(String uploadId, List<MultipartPart> parts)
{
Vector<InputStream> streams = new Vector<>();
for (MultipartPart part : parts)
{
Path partFile = multipartUploadRepo.partFile(uploadId, part.partNumber());
try
{
streams.add(Files.newInputStream(partFile));
}
catch (IOException e)
{
throw new UncheckedIOException(e);
}
}
return new SequenceInputStream(Collections.enumeration(streams));
}
}
@@ -0,0 +1,78 @@
package dev.dinauer.r9t.core.multipart;
import dev.dinauer.r9t.core.object.S3Object;
import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception;
import dev.dinauer.r9t.core.s3.S3Xml;
import jakarta.enterprise.context.ApplicationScoped;
import org.apache.commons.lang3.StringUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
@ApplicationScoped
public class MultipartUploadXmlMapper
{
public String map(MultipartUpload upload)
{
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ "<InitiateMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n"
+ " <Bucket>" + S3Xml.escape(upload.bucket()) + "</Bucket>\n"
+ " <Key>" + S3Xml.escape(upload.key()) + "</Key>\n"
+ " <UploadId>" + S3Xml.escape(upload.uploadId()) + "</UploadId>\n"
+ "</InitiateMultipartUploadResult>";
}
public String map(String bucket, S3Object completed)
{
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ "<CompleteMultipartUploadResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n"
+ " <Bucket>" + S3Xml.escape(bucket) + "</Bucket>\n"
+ " <Key>" + S3Xml.escape(completed.key()) + "</Key>\n"
+ " <ETag>&quot;" + completed.etag() + "&quot;</ETag>\n"
+ "</CompleteMultipartUploadResult>";
}
public List<CompletedPart> map(InputStream body)
{
try
{
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
DocumentBuilder builder = factory.newDocumentBuilder();
Document document = builder.parse(body);
NodeList partNodes = document.getElementsByTagName("Part");
List<CompletedPart> parts = new ArrayList<>();
for (int i = 0; i < partNodes.getLength(); i++)
{
parts.add(toCompletedPart((Element) partNodes.item(i)));
}
return parts;
}
catch (ParserConfigurationException | SAXException | IOException | NumberFormatException | NullPointerException e)
{
throw new S3Exception(S3ErrorCode.MALFORMED_XML, "CompleteMultipartUpload", e);
}
}
private CompletedPart toCompletedPart(Element partElement)
{
int partNumber = Integer.parseInt(text(partElement, "PartNumber"));
String etag = StringUtils.strip(text(partElement, "ETag"), "\"");
return new CompletedPart(partNumber, etag);
}
private String text(Element parent, String tagName)
{
return parent.getElementsByTagName(tagName).item(0).getTextContent();
}
}
@@ -1,6 +1,8 @@
package dev.dinauer.r9t.core.object; package dev.dinauer.r9t.core.object;
import dev.dinauer.r9t.core.bucket.BucketRepo; import dev.dinauer.r9t.core.bucket.BucketRepo;
import dev.dinauer.r9t.core.s3.DigestCopy;
import dev.dinauer.r9t.core.s3.FileAttributes;
import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception; import dev.dinauer.r9t.core.s3.S3Exception;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
@@ -8,26 +10,16 @@ import jakarta.inject.Inject;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.UserDefinedFileAttributeView;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HexFormat;
import java.util.List; import java.util.List;
import java.util.stream.Stream; import java.util.stream.Stream;
@ApplicationScoped @ApplicationScoped
public class ObjectRepo public class ObjectRepo
{ {
private static final String ETAG_ATTRIBUTE = "user.etag";
private static final String CONTENT_TYPE_ATTRIBUTE = "user.contentType";
@Inject @Inject
BucketRepo bucketRepo; BucketRepo bucketRepo;
@@ -53,17 +45,28 @@ public class ObjectRepo
try try
{ {
Files.createDirectories(target.getParent()); Files.createDirectories(target.getParent());
MessageDigest digest = MessageDigest.getInstance("MD5"); String etag = DigestCopy.copyAndHashMd5(data, target);
try (DigestInputStream digestInputStream = new DigestInputStream(data, digest)) FileAttributes.write(target, FileAttributes.ETAG, etag);
{ FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType);
Files.copy(digestInputStream, target, StandardCopyOption.REPLACE_EXISTING);
}
String etag = HexFormat.of().formatHex(digest.digest());
writeAttribute(target, ETAG_ATTRIBUTE, etag);
writeAttribute(target, CONTENT_TYPE_ATTRIBUTE, contentType);
return etag; return etag;
} }
catch (IOException | NoSuchAlgorithmException e) catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e);
}
}
public void storeWithEtag(String bucket, String key, String contentType, InputStream data, String etag)
{
Path target = resolve(bucket, key);
try
{
Files.createDirectories(target.getParent());
Files.copy(data, target, StandardCopyOption.REPLACE_EXISTING);
FileAttributes.write(target, FileAttributes.ETAG, etag);
FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType);
}
catch (IOException e)
{ {
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e);
} }
@@ -75,8 +78,8 @@ public class ObjectRepo
try try
{ {
BasicFileAttributes attributes = Files.readAttributes(target, BasicFileAttributes.class); BasicFileAttributes attributes = Files.readAttributes(target, BasicFileAttributes.class);
String etag = readAttribute(target, ETAG_ATTRIBUTE); String etag = FileAttributes.read(target, FileAttributes.ETAG);
String contentType = readAttribute(target, CONTENT_TYPE_ATTRIBUTE); String contentType = FileAttributes.read(target, FileAttributes.CONTENT_TYPE);
return new S3Object(key, attributes.size(), attributes.lastModifiedTime().toInstant(), etag, contentType); return new S3Object(key, attributes.size(), attributes.lastModifiedTime().toInstant(), etag, contentType);
} }
catch (IOException e) catch (IOException e)
@@ -117,19 +120,4 @@ public class ObjectRepo
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e);
} }
} }
private void writeAttribute(Path file, String name, String value) throws IOException
{
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
view.write(name, StandardCharsets.UTF_8.encode(value));
}
private String readAttribute(Path file, String name) throws IOException
{
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
ByteBuffer buffer = ByteBuffer.allocate(view.size(name));
view.read(name, buffer);
buffer.flip();
return StandardCharsets.UTF_8.decode(buffer).toString();
}
} }
@@ -1,6 +1,10 @@
package dev.dinauer.r9t.core.object; package dev.dinauer.r9t.core.object;
import dev.dinauer.r9t.core.auth.AwsChunkedInputStream; import dev.dinauer.r9t.core.auth.AwsChunkedInputStream;
import dev.dinauer.r9t.core.multipart.CompletedPart;
import dev.dinauer.r9t.core.multipart.MultipartUpload;
import dev.dinauer.r9t.core.multipart.MultipartUploadService;
import dev.dinauer.r9t.core.multipart.MultipartUploadXmlMapper;
import io.quarkus.security.Authenticated; import io.quarkus.security.Authenticated;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import jakarta.inject.Inject; import jakarta.inject.Inject;
@@ -9,15 +13,20 @@ import jakarta.ws.rs.DELETE;
import jakarta.ws.rs.GET; import jakarta.ws.rs.GET;
import jakarta.ws.rs.HEAD; import jakarta.ws.rs.HEAD;
import jakarta.ws.rs.HeaderParam; import jakarta.ws.rs.HeaderParam;
import jakarta.ws.rs.POST;
import jakarta.ws.rs.PUT; import jakarta.ws.rs.PUT;
import jakarta.ws.rs.Path; import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam; import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces; import jakarta.ws.rs.Produces;
import jakarta.ws.rs.QueryParam;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.UriInfo;
import java.io.InputStream; import java.io.InputStream;
import java.util.Date; import java.util.Date;
import java.util.List;
@Path("/{bucket}") @Path("/{bucket}")
@Authenticated @Authenticated
@@ -29,6 +38,12 @@ public class ObjectResource
@Inject @Inject
ObjectXmlMapper objectXmlMapper; ObjectXmlMapper objectXmlMapper;
@Inject
MultipartUploadService multipartUploadService;
@Inject
MultipartUploadXmlMapper multipartUploadXmlMapper;
@GET @GET
@Produces(MediaType.APPLICATION_XML) @Produces(MediaType.APPLICATION_XML)
public Response list(@PathParam("bucket") String bucket) public Response list(@PathParam("bucket") String bucket)
@@ -40,6 +55,7 @@ public class ObjectResource
@Path("{key:.+}") @Path("{key:.+}")
@Consumes(MediaType.WILDCARD) @Consumes(MediaType.WILDCARD)
public Response put(@PathParam("bucket") String bucket, @PathParam("key") String key, public Response put(@PathParam("bucket") String bucket, @PathParam("key") String key,
@QueryParam("uploadId") String uploadId, @QueryParam("partNumber") Integer partNumber,
@HeaderParam("Content-Type") String contentType, @HeaderParam("Content-Type") String contentType,
@HeaderParam("X-Amz-Content-Sha256") String contentSha256, InputStream body) @HeaderParam("X-Amz-Content-Sha256") String contentSha256, InputStream body)
{ {
@@ -48,10 +64,33 @@ public class ObjectResource
{ {
payload = new AwsChunkedInputStream(body); payload = new AwsChunkedInputStream(body);
} }
if (uploadId != null)
{
String partEtag = multipartUploadService.uploadPart(bucket, key, uploadId, partNumber, payload);
return Response.ok().header("ETag", "\"" + partEtag + "\"").build();
}
String etag = objectService.put(bucket, key, contentType, payload); String etag = objectService.put(bucket, key, contentType, payload);
return Response.ok().header("ETag", "\"" + etag + "\"").build(); return Response.ok().header("ETag", "\"" + etag + "\"").build();
} }
@POST
@Path("{key:.+}")
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.APPLICATION_XML)
public Response post(@PathParam("bucket") String bucket, @PathParam("key") String key,
@Context UriInfo uriInfo, @QueryParam("uploadId") String uploadId,
@HeaderParam("Content-Type") String contentType, InputStream body)
{
if (uriInfo.getQueryParameters().containsKey("uploads"))
{
MultipartUpload upload = multipartUploadService.initiate(bucket, key, contentType);
return Response.ok(multipartUploadXmlMapper.map(upload)).build();
}
List<CompletedPart> parts = multipartUploadXmlMapper.map(body);
S3Object completed = multipartUploadService.complete(bucket, key, uploadId, parts);
return Response.ok(multipartUploadXmlMapper.map(bucket, completed)).build();
}
@GET @GET
@Path("{key:.+}") @Path("{key:.+}")
public Response get(@PathParam("bucket") String bucket, @PathParam("key") String key) public Response get(@PathParam("bucket") String bucket, @PathParam("key") String key)
@@ -79,8 +118,14 @@ public class ObjectResource
@DELETE @DELETE
@Path("{key:.+}") @Path("{key:.+}")
public Response delete(@PathParam("bucket") String bucket, @PathParam("key") String key) public Response delete(@PathParam("bucket") String bucket, @PathParam("key") String key,
@QueryParam("uploadId") String uploadId)
{ {
if (uploadId != null)
{
multipartUploadService.abort(bucket, key, uploadId);
return Response.noContent().build();
}
objectService.delete(bucket, key); objectService.delete(bucket, key);
return Response.noContent().build(); return Response.noContent().build();
} }
@@ -0,0 +1,40 @@
package dev.dinauer.r9t.core.s3;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.HexFormat;
public final class DigestCopy
{
private DigestCopy()
{
}
public static String copyAndHashMd5(InputStream data, Path target) throws IOException
{
MessageDigest digest = newMd5();
try (DigestInputStream digestInputStream = new DigestInputStream(data, digest))
{
Files.copy(digestInputStream, target, StandardCopyOption.REPLACE_EXISTING);
}
return HexFormat.of().formatHex(digest.digest());
}
public static MessageDigest newMd5()
{
try
{
return MessageDigest.getInstance("MD5");
}
catch (NoSuchAlgorithmException e)
{
throw new IllegalStateException(e);
}
}
}
@@ -0,0 +1,33 @@
package dev.dinauer.r9t.core.s3;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.UserDefinedFileAttributeView;
public final class FileAttributes
{
public static final String ETAG = "user.etag";
public static final String CONTENT_TYPE = "user.contentType";
private FileAttributes()
{
}
public static void write(Path file, String name, String value) throws IOException
{
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
view.write(name, StandardCharsets.UTF_8.encode(value));
}
public static String read(Path file, String name) throws IOException
{
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
ByteBuffer buffer = ByteBuffer.allocate(view.size(name));
view.read(name, buffer);
buffer.flip();
return StandardCharsets.UTF_8.decode(buffer).toString();
}
}
@@ -8,6 +8,9 @@ public enum S3ErrorCode
BUCKET_NOT_EMPTY("BucketNotEmpty", 409, "The bucket you tried to delete is not empty"), BUCKET_NOT_EMPTY("BucketNotEmpty", 409, "The bucket you tried to delete is not empty"),
INVALID_BUCKET_NAME("InvalidBucketName", 400, "The specified bucket is not valid"), INVALID_BUCKET_NAME("InvalidBucketName", 400, "The specified bucket is not valid"),
INVALID_ARGUMENT("InvalidArgument", 400, "Invalid argument"), INVALID_ARGUMENT("InvalidArgument", 400, "Invalid argument"),
NO_SUCH_UPLOAD("NoSuchUpload", 404, "The specified multipart upload does not exist"),
INVALID_PART("InvalidPart", 400, "One or more of the specified parts could not be found"),
MALFORMED_XML("MalformedXML", 400, "The XML you provided was not well-formed or did not validate against our published schema"),
INTERNAL_ERROR("InternalError", 500, "We encountered an internal error"); INTERNAL_ERROR("InternalError", 500, "We encountered an internal error");
private final String code; private final String code;
+1 -2
View File
@@ -1,2 +1 @@
%dev.dev.dinauer.r9t.credentials.access-key=devkey quarkus.http.limits.max-body-size=5G
%dev.dev.dinauer.r9t.credentials.secret-key=devsecret
@@ -0,0 +1,142 @@
package dev.dinauer.r9t.core.multipart;
import dev.dinauer.r9t.core.auth.SigV4TestSigner;
import io.quarkus.test.common.http.TestHTTPResource;
import io.quarkus.test.junit.QuarkusTest;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
public class MultipartUploadTest
{
private static final String BUCKET = "multipart-test-bucket";
private static final String KEY = "multipart-test.txt";
private static final String ACCESS_KEY = "testkey";
private static final String SECRET_KEY = "testsecret";
@TestHTTPResource
URI baseUri;
private final HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build();
@Test
public void completeUploadAssemblesObject() throws Exception
{
createBucket();
HttpResponse<String> initiate = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploads="),
new byte[0], "text/plain");
assertEquals(200, initiate.statusCode());
String uploadId = extract(initiate.body(), "UploadId");
String etag1 = uploadPart(uploadId, 1, "hello ");
String etag2 = uploadPart(uploadId, 2, "world");
String completeBody = "<CompleteMultipartUpload>"
+ "<Part><PartNumber>1</PartNumber><ETag>\"" + etag1 + "\"</ETag></Part>"
+ "<Part><PartNumber>2</PartNumber><ETag>\"" + etag2 + "\"</ETag></Part>"
+ "</CompleteMultipartUpload>";
HttpResponse<String> complete = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploadId=" + uploadId),
completeBody.getBytes(StandardCharsets.UTF_8), null);
assertEquals(200, complete.statusCode());
String finalEtag = extract(complete.body(), "ETag");
assertTrue(Pattern.matches("[0-9a-f]{32}-2", finalEtag), "unexpected ETag: " + finalEtag);
HttpResponse<byte[]> get = sendForBytes("GET", baseUri.resolve(BUCKET + "/" + KEY));
assertEquals(200, get.statusCode());
assertEquals("text/plain", get.headers().firstValue("Content-Type").orElse(""));
assertEquals("hello world", new String(get.body(), StandardCharsets.UTF_8));
}
@Test
public void abortDiscardsStagedParts() throws Exception
{
createBucket();
HttpResponse<String> initiate = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploads="),
new byte[0], "text/plain");
String uploadId = extract(initiate.body(), "UploadId");
uploadPart(uploadId, 1, "hello ");
HttpResponse<Void> abort = send("DELETE", baseUri.resolve(BUCKET + "/" + KEY + "?uploadId=" + uploadId),
new byte[0], null);
assertEquals(204, abort.statusCode());
HttpResponse<Void> uploadAfterAbort = send("PUT",
baseUri.resolve(BUCKET + "/" + KEY + "?partNumber=2&uploadId=" + uploadId), "world".getBytes(StandardCharsets.UTF_8), null);
assertEquals(404, uploadAfterAbort.statusCode());
}
@AfterEach
public void cleanup() throws Exception
{
send("DELETE", baseUri.resolve(BUCKET + "/" + KEY), new byte[0], null);
send("DELETE", baseUri.resolve(BUCKET), new byte[0], null);
}
private void createBucket() throws Exception
{
HttpResponse<Void> createBucket = send("PUT", baseUri.resolve(BUCKET), new byte[0], null);
assertEquals(200, createBucket.statusCode());
}
private String uploadPart(String uploadId, int partNumber, String content) throws Exception
{
byte[] body = content.getBytes(StandardCharsets.UTF_8);
HttpResponse<Void> put = send("PUT",
baseUri.resolve(BUCKET + "/" + KEY + "?partNumber=" + partNumber + "&uploadId=" + uploadId), body, null);
assertEquals(200, put.statusCode());
return put.headers().firstValue("ETag").orElseThrow().replace("\"", "");
}
private String extract(String xml, String tag)
{
Matcher matcher = Pattern.compile("<" + tag + ">(.*?)</" + tag + ">").matcher(xml);
assertTrue(matcher.find(), "missing <" + tag + "> in: " + xml);
return matcher.group(1).replace("&quot;", "").replace("\"", "");
}
private HttpResponse<Void> send(String method, URI uri, byte[] body, String contentType) throws Exception
{
HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
.method(method, HttpRequest.BodyPublishers.ofByteArray(body));
if (contentType != null)
{
builder.header("Content-Type", contentType);
}
SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body);
return client.send(builder.build(), HttpResponse.BodyHandlers.discarding());
}
private HttpResponse<String> sendForString(String method, URI uri, byte[] body, String contentType) throws Exception
{
HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
.method(method, HttpRequest.BodyPublishers.ofByteArray(body));
if (contentType != null)
{
builder.header("Content-Type", contentType);
}
SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body);
return client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
}
private HttpResponse<byte[]> sendForBytes(String method, URI uri) throws Exception
{
HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
.method(method, HttpRequest.BodyPublishers.ofByteArray(new byte[0]));
SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, new byte[0]);
return client.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray());
}
}