From 0b8e36fe58176b8cfcd34ff9a52db914b9c697cd Mon Sep 17 00:00:00 2001 From: Andreas Dinauer Date: Fri, 21 Aug 2026 15:14:32 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20chunked=20uploads?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dinauer/r9t/core/bucket/BucketRepo.java | 1 + .../dinauer/r9t/core/init/InitService.java | 10 +- .../r9t/core/multipart/CompletedPart.java | 5 + .../r9t/core/multipart/MultipartPart.java | 5 + .../r9t/core/multipart/MultipartUpload.java | 7 + .../core/multipart/MultipartUploadRepo.java | 152 ++++++++++++++++++ .../multipart/MultipartUploadService.java | 152 ++++++++++++++++++ .../multipart/MultipartUploadXmlMapper.java | 78 +++++++++ .../dinauer/r9t/core/object/ObjectRepo.java | 60 +++---- .../r9t/core/object/ObjectResource.java | 47 +++++- .../dev/dinauer/r9t/core/s3/DigestCopy.java | 40 +++++ .../dinauer/r9t/core/s3/FileAttributes.java | 33 ++++ .../dev/dinauer/r9t/core/s3/S3ErrorCode.java | 3 + src/main/resources/application.properties | 3 +- .../core/multipart/MultipartUploadTest.java | 142 ++++++++++++++++ 15 files changed, 698 insertions(+), 40 deletions(-) create mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/CompletedPart.java create mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/MultipartPart.java create mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/MultipartUpload.java create mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java create mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java create mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadXmlMapper.java create mode 100644 src/main/java/dev/dinauer/r9t/core/s3/DigestCopy.java create mode 100644 src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java create mode 100644 src/test/java/dev/dinauer/r9t/core/multipart/MultipartUploadTest.java diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java index 8052e8e..dc69cd4 100644 --- a/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java @@ -87,6 +87,7 @@ public class BucketRepo { return entries .filter(Files::isDirectory) + .filter(path -> !path.getFileName().toString().startsWith(".")) .map(this::toBucket) .toList(); } diff --git a/src/main/java/dev/dinauer/r9t/core/init/InitService.java b/src/main/java/dev/dinauer/r9t/core/init/InitService.java index 755d0e1..d832c95 100644 --- a/src/main/java/dev/dinauer/r9t/core/init/InitService.java +++ b/src/main/java/dev/dinauer/r9t/core/init/InitService.java @@ -6,6 +6,7 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Observes; import jakarta.inject.Inject; import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; import java.util.List; import java.util.Optional; @@ -13,6 +14,8 @@ import java.util.Optional; @ApplicationScoped public class InitService { + private static final Logger LOG = Logger.getLogger(InitService.class); + @ConfigProperty(name = "dev.dinauer.r9t.init.buckets") Optional> buckets; @@ -23,9 +26,14 @@ public class InitService { for (String bucket : buckets.orElse(List.of())) { - if (!bucketService.exists(bucket)) + if (bucketService.exists(bucket)) + { + LOG.infof("Bootstrap bucket '%s' already exists, skipping", bucket); + } + else { bucketService.create(bucket); + LOG.infof("Bootstrapped bucket '%s'", bucket); } } } diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/CompletedPart.java b/src/main/java/dev/dinauer/r9t/core/multipart/CompletedPart.java new file mode 100644 index 0000000..d0eecee --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/multipart/CompletedPart.java @@ -0,0 +1,5 @@ +package dev.dinauer.r9t.core.multipart; + +public record CompletedPart(int partNumber, String etag) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartPart.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartPart.java new file mode 100644 index 0000000..f5e3178 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartPart.java @@ -0,0 +1,5 @@ +package dev.dinauer.r9t.core.multipart; + +public record MultipartPart(int partNumber, String etag, long size) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUpload.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUpload.java new file mode 100644 index 0000000..16bf065 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUpload.java @@ -0,0 +1,7 @@ +package dev.dinauer.r9t.core.multipart; + +import java.time.Instant; + +public record MultipartUpload(String uploadId, String bucket, String key, String contentType, Instant initiated) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java new file mode 100644 index 0000000..fbda3da --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java @@ -0,0 +1,152 @@ +package dev.dinauer.r9t.core.multipart; + +import dev.dinauer.r9t.core.bucket.BucketRepo; +import dev.dinauer.r9t.core.s3.DigestCopy; +import dev.dinauer.r9t.core.s3.FileAttributes; +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.S3Exception; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Comparator; +import java.util.UUID; +import java.util.stream.Stream; + +@ApplicationScoped +public class MultipartUploadRepo +{ + private static final String BUCKET_ATTRIBUTE = "user.bucket"; + private static final String KEY_ATTRIBUTE = "user.key"; + private static final String INITIATED_ATTRIBUTE = "user.initiated"; + + @Inject + BucketRepo bucketRepo; + + public Path resolve(String uploadId) + { + Path root = stagingRoot(); + Path target = root.resolve(uploadId).normalize(); + if (!target.startsWith(root)) + { + throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, uploadId); + } + return target; + } + + public boolean exists(String uploadId) + { + return Files.isDirectory(resolve(uploadId)); + } + + public MultipartUpload create(String bucket, String key, String contentType) + { + String uploadId = UUID.randomUUID().toString().replace("-", ""); + Path dir = resolve(uploadId); + Instant initiated = Instant.now(); + try + { + Files.createDirectories(dir); + FileAttributes.write(dir, BUCKET_ATTRIBUTE, bucket); + FileAttributes.write(dir, KEY_ATTRIBUTE, key); + FileAttributes.write(dir, FileAttributes.CONTENT_TYPE, contentType); + FileAttributes.write(dir, INITIATED_ATTRIBUTE, initiated.toString()); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); + } + return new MultipartUpload(uploadId, bucket, key, contentType, initiated); + } + + public MultipartUpload metadata(String uploadId) + { + Path dir = resolve(uploadId); + try + { + String bucket = FileAttributes.read(dir, BUCKET_ATTRIBUTE); + String key = FileAttributes.read(dir, KEY_ATTRIBUTE); + String contentType = FileAttributes.read(dir, FileAttributes.CONTENT_TYPE); + Instant initiated = Instant.parse(FileAttributes.read(dir, INITIATED_ATTRIBUTE)); + return new MultipartUpload(uploadId, bucket, key, contentType, initiated); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); + } + } + + public String storePart(String uploadId, int partNumber, InputStream data) + { + Path partFile = partFile(uploadId, partNumber); + try + { + String etag = DigestCopy.copyAndHashMd5(data, partFile); + FileAttributes.write(partFile, FileAttributes.ETAG, etag); + return etag; + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); + } + } + + public boolean partExists(String uploadId, int partNumber) + { + return Files.isRegularFile(partFile(uploadId, partNumber)); + } + + public MultipartPart part(String uploadId, int partNumber) + { + Path partFile = partFile(uploadId, partNumber); + try + { + String etag = FileAttributes.read(partFile, FileAttributes.ETAG); + long size = Files.size(partFile); + return new MultipartPart(partNumber, etag, size); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); + } + } + + public Path partFile(String uploadId, int partNumber) + { + return resolve(uploadId).resolve(String.format("%05d", partNumber)); + } + + public void delete(String uploadId) + { + Path dir = resolve(uploadId); + try (Stream entries = Files.walk(dir)) + { + for (Path path : entries.sorted(Comparator.reverseOrder()).toList()) + { + Files.deleteIfExists(path); + } + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); + } + } + + private Path stagingRoot() + { + Path root = bucketRepo.root().resolve(".multipart"); + try + { + Files.createDirectories(root); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, root.toString(), e); + } + return root; + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java new file mode 100644 index 0000000..f540ee5 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java @@ -0,0 +1,152 @@ +package dev.dinauer.r9t.core.multipart; + +import dev.dinauer.r9t.core.bucket.BucketService; +import dev.dinauer.r9t.core.object.ObjectRepo; +import dev.dinauer.r9t.core.object.S3Object; +import dev.dinauer.r9t.core.s3.DigestCopy; +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.S3Exception; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HexFormat; +import java.util.List; +import java.util.Vector; + +@ApplicationScoped +public class MultipartUploadService +{ + @Inject + MultipartUploadRepo multipartUploadRepo; + + @Inject + ObjectRepo objectRepo; + + @Inject + BucketService bucketService; + + public MultipartUpload initiate(String bucket, String key, String contentType) + { + bucketService.require(bucket); + String resolvedContentType = contentType; + if (resolvedContentType == null) + { + resolvedContentType = "application/octet-stream"; + } + return multipartUploadRepo.create(bucket, key, resolvedContentType); + } + + public String uploadPart(String bucket, String key, String uploadId, Integer partNumber, InputStream data) + { + require(bucket, key, uploadId); + if (partNumber == null || partNumber < 1 || partNumber > 10000) + { + throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, "partNumber"); + } + return multipartUploadRepo.storePart(uploadId, partNumber, data); + } + + public S3Object complete(String bucket, String key, String uploadId, List parts) + { + MultipartUpload upload = require(bucket, key, uploadId); + if (parts.isEmpty()) + { + throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, uploadId); + } + List resolvedParts = resolveParts(uploadId, parts); + String etag = combineEtags(resolvedParts); + try (InputStream assembled = assemble(uploadId, resolvedParts)) + { + objectRepo.storeWithEtag(bucket, key, upload.contentType(), assembled, etag); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); + } + multipartUploadRepo.delete(uploadId); + return objectRepo.metadata(bucket, key); + } + + public void abort(String bucket, String key, String uploadId) + { + require(bucket, key, uploadId); + multipartUploadRepo.delete(uploadId); + } + + private MultipartUpload require(String bucket, String key, String uploadId) + { + bucketService.require(bucket); + if (!multipartUploadRepo.exists(uploadId)) + { + throw new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD, uploadId); + } + MultipartUpload upload = multipartUploadRepo.metadata(uploadId); + if (!upload.bucket().equals(bucket) || !upload.key().equals(key)) + { + throw new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD, uploadId); + } + return upload; + } + + private List resolveParts(String uploadId, List parts) + { + List resolved = new ArrayList<>(); + int previousPartNumber = 0; + for (CompletedPart requested : parts) + { + if (requested.partNumber() <= previousPartNumber) + { + throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId); + } + if (!multipartUploadRepo.partExists(uploadId, requested.partNumber())) + { + throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId); + } + MultipartPart stored = multipartUploadRepo.part(uploadId, requested.partNumber()); + if (!stored.etag().equalsIgnoreCase(requested.etag())) + { + throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId); + } + resolved.add(stored); + previousPartNumber = requested.partNumber(); + } + return resolved; + } + + private String combineEtags(List parts) + { + MessageDigest digest = DigestCopy.newMd5(); + for (MultipartPart part : parts) + { + digest.update(HexFormat.of().parseHex(part.etag())); + } + return HexFormat.of().formatHex(digest.digest()) + "-" + parts.size(); + } + + private InputStream assemble(String uploadId, List parts) + { + Vector streams = new Vector<>(); + for (MultipartPart part : parts) + { + Path partFile = multipartUploadRepo.partFile(uploadId, part.partNumber()); + try + { + streams.add(Files.newInputStream(partFile)); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + return new SequenceInputStream(Collections.enumeration(streams)); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadXmlMapper.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadXmlMapper.java new file mode 100644 index 0000000..77eabcd --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadXmlMapper.java @@ -0,0 +1,78 @@ +package dev.dinauer.r9t.core.multipart; + +import dev.dinauer.r9t.core.object.S3Object; +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.S3Exception; +import dev.dinauer.r9t.core.s3.S3Xml; +import jakarta.enterprise.context.ApplicationScoped; +import org.apache.commons.lang3.StringUtils; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +@ApplicationScoped +public class MultipartUploadXmlMapper +{ + public String map(MultipartUpload upload) + { + return "\n" + + "\n" + + " " + S3Xml.escape(upload.bucket()) + "\n" + + " " + S3Xml.escape(upload.key()) + "\n" + + " " + S3Xml.escape(upload.uploadId()) + "\n" + + ""; + } + + public String map(String bucket, S3Object completed) + { + return "\n" + + "\n" + + " " + S3Xml.escape(bucket) + "\n" + + " " + S3Xml.escape(completed.key()) + "\n" + + " "" + completed.etag() + ""\n" + + ""; + } + + public List map(InputStream body) + { + try + { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(body); + NodeList partNodes = document.getElementsByTagName("Part"); + List parts = new ArrayList<>(); + for (int i = 0; i < partNodes.getLength(); i++) + { + parts.add(toCompletedPart((Element) partNodes.item(i))); + } + return parts; + } + catch (ParserConfigurationException | SAXException | IOException | NumberFormatException | NullPointerException e) + { + throw new S3Exception(S3ErrorCode.MALFORMED_XML, "CompleteMultipartUpload", e); + } + } + + private CompletedPart toCompletedPart(Element partElement) + { + int partNumber = Integer.parseInt(text(partElement, "PartNumber")); + String etag = StringUtils.strip(text(partElement, "ETag"), "\""); + return new CompletedPart(partNumber, etag); + } + + private String text(Element parent, String tagName) + { + return parent.getElementsByTagName(tagName).item(0).getTextContent(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java index 5182781..e14eb4c 100644 --- a/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java @@ -1,6 +1,8 @@ package dev.dinauer.r9t.core.object; import dev.dinauer.r9t.core.bucket.BucketRepo; +import dev.dinauer.r9t.core.s3.DigestCopy; +import dev.dinauer.r9t.core.s3.FileAttributes; import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3Exception; import jakarta.enterprise.context.ApplicationScoped; @@ -8,26 +10,16 @@ import jakarta.inject.Inject; import java.io.IOException; import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.UserDefinedFileAttributeView; -import java.security.DigestInputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.HexFormat; import java.util.List; import java.util.stream.Stream; @ApplicationScoped public class ObjectRepo { - private static final String ETAG_ATTRIBUTE = "user.etag"; - private static final String CONTENT_TYPE_ATTRIBUTE = "user.contentType"; - @Inject BucketRepo bucketRepo; @@ -53,17 +45,28 @@ public class ObjectRepo try { Files.createDirectories(target.getParent()); - MessageDigest digest = MessageDigest.getInstance("MD5"); - try (DigestInputStream digestInputStream = new DigestInputStream(data, digest)) - { - Files.copy(digestInputStream, target, StandardCopyOption.REPLACE_EXISTING); - } - String etag = HexFormat.of().formatHex(digest.digest()); - writeAttribute(target, ETAG_ATTRIBUTE, etag); - writeAttribute(target, CONTENT_TYPE_ATTRIBUTE, contentType); + String etag = DigestCopy.copyAndHashMd5(data, target); + FileAttributes.write(target, FileAttributes.ETAG, etag); + FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType); return etag; } - catch (IOException | NoSuchAlgorithmException e) + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); + } + } + + public void storeWithEtag(String bucket, String key, String contentType, InputStream data, String etag) + { + Path target = resolve(bucket, key); + try + { + Files.createDirectories(target.getParent()); + Files.copy(data, target, StandardCopyOption.REPLACE_EXISTING); + FileAttributes.write(target, FileAttributes.ETAG, etag); + FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType); + } + catch (IOException e) { throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); } @@ -75,8 +78,8 @@ public class ObjectRepo try { BasicFileAttributes attributes = Files.readAttributes(target, BasicFileAttributes.class); - String etag = readAttribute(target, ETAG_ATTRIBUTE); - String contentType = readAttribute(target, CONTENT_TYPE_ATTRIBUTE); + String etag = FileAttributes.read(target, FileAttributes.ETAG); + String contentType = FileAttributes.read(target, FileAttributes.CONTENT_TYPE); return new S3Object(key, attributes.size(), attributes.lastModifiedTime().toInstant(), etag, contentType); } catch (IOException e) @@ -117,19 +120,4 @@ public class ObjectRepo throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); } } - - private void writeAttribute(Path file, String name, String value) throws IOException - { - UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); - view.write(name, StandardCharsets.UTF_8.encode(value)); - } - - private String readAttribute(Path file, String name) throws IOException - { - UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); - ByteBuffer buffer = ByteBuffer.allocate(view.size(name)); - view.read(name, buffer); - buffer.flip(); - return StandardCharsets.UTF_8.decode(buffer).toString(); - } } diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java index edc52e5..d1eaaeb 100644 --- a/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java @@ -1,6 +1,10 @@ package dev.dinauer.r9t.core.object; import dev.dinauer.r9t.core.auth.AwsChunkedInputStream; +import dev.dinauer.r9t.core.multipart.CompletedPart; +import dev.dinauer.r9t.core.multipart.MultipartUpload; +import dev.dinauer.r9t.core.multipart.MultipartUploadService; +import dev.dinauer.r9t.core.multipart.MultipartUploadXmlMapper; import io.quarkus.security.Authenticated; import org.apache.commons.lang3.StringUtils; import jakarta.inject.Inject; @@ -9,15 +13,20 @@ import jakarta.ws.rs.DELETE; import jakarta.ws.rs.GET; import jakarta.ws.rs.HEAD; import jakarta.ws.rs.HeaderParam; +import jakarta.ws.rs.POST; import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; import java.io.InputStream; import java.util.Date; +import java.util.List; @Path("/{bucket}") @Authenticated @@ -29,6 +38,12 @@ public class ObjectResource @Inject ObjectXmlMapper objectXmlMapper; + @Inject + MultipartUploadService multipartUploadService; + + @Inject + MultipartUploadXmlMapper multipartUploadXmlMapper; + @GET @Produces(MediaType.APPLICATION_XML) public Response list(@PathParam("bucket") String bucket) @@ -40,6 +55,7 @@ public class ObjectResource @Path("{key:.+}") @Consumes(MediaType.WILDCARD) public Response put(@PathParam("bucket") String bucket, @PathParam("key") String key, + @QueryParam("uploadId") String uploadId, @QueryParam("partNumber") Integer partNumber, @HeaderParam("Content-Type") String contentType, @HeaderParam("X-Amz-Content-Sha256") String contentSha256, InputStream body) { @@ -48,10 +64,33 @@ public class ObjectResource { payload = new AwsChunkedInputStream(body); } + if (uploadId != null) + { + String partEtag = multipartUploadService.uploadPart(bucket, key, uploadId, partNumber, payload); + return Response.ok().header("ETag", "\"" + partEtag + "\"").build(); + } String etag = objectService.put(bucket, key, contentType, payload); return Response.ok().header("ETag", "\"" + etag + "\"").build(); } + @POST + @Path("{key:.+}") + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_XML) + public Response post(@PathParam("bucket") String bucket, @PathParam("key") String key, + @Context UriInfo uriInfo, @QueryParam("uploadId") String uploadId, + @HeaderParam("Content-Type") String contentType, InputStream body) + { + if (uriInfo.getQueryParameters().containsKey("uploads")) + { + MultipartUpload upload = multipartUploadService.initiate(bucket, key, contentType); + return Response.ok(multipartUploadXmlMapper.map(upload)).build(); + } + List parts = multipartUploadXmlMapper.map(body); + S3Object completed = multipartUploadService.complete(bucket, key, uploadId, parts); + return Response.ok(multipartUploadXmlMapper.map(bucket, completed)).build(); + } + @GET @Path("{key:.+}") public Response get(@PathParam("bucket") String bucket, @PathParam("key") String key) @@ -79,8 +118,14 @@ public class ObjectResource @DELETE @Path("{key:.+}") - public Response delete(@PathParam("bucket") String bucket, @PathParam("key") String key) + public Response delete(@PathParam("bucket") String bucket, @PathParam("key") String key, + @QueryParam("uploadId") String uploadId) { + if (uploadId != null) + { + multipartUploadService.abort(bucket, key, uploadId); + return Response.noContent().build(); + } objectService.delete(bucket, key); return Response.noContent().build(); } diff --git a/src/main/java/dev/dinauer/r9t/core/s3/DigestCopy.java b/src/main/java/dev/dinauer/r9t/core/s3/DigestCopy.java new file mode 100644 index 0000000..ff5303d --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/s3/DigestCopy.java @@ -0,0 +1,40 @@ +package dev.dinauer.r9t.core.s3; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +public final class DigestCopy +{ + private DigestCopy() + { + } + + public static String copyAndHashMd5(InputStream data, Path target) throws IOException + { + MessageDigest digest = newMd5(); + try (DigestInputStream digestInputStream = new DigestInputStream(data, digest)) + { + Files.copy(digestInputStream, target, StandardCopyOption.REPLACE_EXISTING); + } + return HexFormat.of().formatHex(digest.digest()); + } + + public static MessageDigest newMd5() + { + try + { + return MessageDigest.getInstance("MD5"); + } + catch (NoSuchAlgorithmException e) + { + throw new IllegalStateException(e); + } + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java b/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java new file mode 100644 index 0000000..9af5b59 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java @@ -0,0 +1,33 @@ +package dev.dinauer.r9t.core.s3; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.UserDefinedFileAttributeView; + +public final class FileAttributes +{ + public static final String ETAG = "user.etag"; + public static final String CONTENT_TYPE = "user.contentType"; + + private FileAttributes() + { + } + + public static void write(Path file, String name, String value) throws IOException + { + UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); + view.write(name, StandardCharsets.UTF_8.encode(value)); + } + + public static String read(Path file, String name) throws IOException + { + UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); + ByteBuffer buffer = ByteBuffer.allocate(view.size(name)); + view.read(name, buffer); + buffer.flip(); + return StandardCharsets.UTF_8.decode(buffer).toString(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java b/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java index a7d8582..75f5616 100644 --- a/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java +++ b/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java @@ -8,6 +8,9 @@ public enum S3ErrorCode BUCKET_NOT_EMPTY("BucketNotEmpty", 409, "The bucket you tried to delete is not empty"), INVALID_BUCKET_NAME("InvalidBucketName", 400, "The specified bucket is not valid"), INVALID_ARGUMENT("InvalidArgument", 400, "Invalid argument"), + NO_SUCH_UPLOAD("NoSuchUpload", 404, "The specified multipart upload does not exist"), + INVALID_PART("InvalidPart", 400, "One or more of the specified parts could not be found"), + MALFORMED_XML("MalformedXML", 400, "The XML you provided was not well-formed or did not validate against our published schema"), INTERNAL_ERROR("InternalError", 500, "We encountered an internal error"); private final String code; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 025cd02..e864c2d 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1,2 +1 @@ -%dev.dev.dinauer.r9t.credentials.access-key=devkey -%dev.dev.dinauer.r9t.credentials.secret-key=devsecret +quarkus.http.limits.max-body-size=5G \ No newline at end of file diff --git a/src/test/java/dev/dinauer/r9t/core/multipart/MultipartUploadTest.java b/src/test/java/dev/dinauer/r9t/core/multipart/MultipartUploadTest.java new file mode 100644 index 0000000..3803ad3 --- /dev/null +++ b/src/test/java/dev/dinauer/r9t/core/multipart/MultipartUploadTest.java @@ -0,0 +1,142 @@ +package dev.dinauer.r9t.core.multipart; + +import dev.dinauer.r9t.core.auth.SigV4TestSigner; +import io.quarkus.test.common.http.TestHTTPResource; +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@QuarkusTest +public class MultipartUploadTest +{ + private static final String BUCKET = "multipart-test-bucket"; + private static final String KEY = "multipart-test.txt"; + private static final String ACCESS_KEY = "testkey"; + private static final String SECRET_KEY = "testsecret"; + + @TestHTTPResource + URI baseUri; + + private final HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); + + @Test + public void completeUploadAssemblesObject() throws Exception + { + createBucket(); + + HttpResponse initiate = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploads="), + new byte[0], "text/plain"); + assertEquals(200, initiate.statusCode()); + String uploadId = extract(initiate.body(), "UploadId"); + + String etag1 = uploadPart(uploadId, 1, "hello "); + String etag2 = uploadPart(uploadId, 2, "world"); + + String completeBody = "" + + "1\"" + etag1 + "\"" + + "2\"" + etag2 + "\"" + + ""; + HttpResponse complete = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploadId=" + uploadId), + completeBody.getBytes(StandardCharsets.UTF_8), null); + assertEquals(200, complete.statusCode()); + String finalEtag = extract(complete.body(), "ETag"); + assertTrue(Pattern.matches("[0-9a-f]{32}-2", finalEtag), "unexpected ETag: " + finalEtag); + + HttpResponse get = sendForBytes("GET", baseUri.resolve(BUCKET + "/" + KEY)); + assertEquals(200, get.statusCode()); + assertEquals("text/plain", get.headers().firstValue("Content-Type").orElse("")); + assertEquals("hello world", new String(get.body(), StandardCharsets.UTF_8)); + } + + @Test + public void abortDiscardsStagedParts() throws Exception + { + createBucket(); + + HttpResponse initiate = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploads="), + new byte[0], "text/plain"); + String uploadId = extract(initiate.body(), "UploadId"); + + uploadPart(uploadId, 1, "hello "); + + HttpResponse abort = send("DELETE", baseUri.resolve(BUCKET + "/" + KEY + "?uploadId=" + uploadId), + new byte[0], null); + assertEquals(204, abort.statusCode()); + + HttpResponse uploadAfterAbort = send("PUT", + baseUri.resolve(BUCKET + "/" + KEY + "?partNumber=2&uploadId=" + uploadId), "world".getBytes(StandardCharsets.UTF_8), null); + assertEquals(404, uploadAfterAbort.statusCode()); + } + + @AfterEach + public void cleanup() throws Exception + { + send("DELETE", baseUri.resolve(BUCKET + "/" + KEY), new byte[0], null); + send("DELETE", baseUri.resolve(BUCKET), new byte[0], null); + } + + private void createBucket() throws Exception + { + HttpResponse createBucket = send("PUT", baseUri.resolve(BUCKET), new byte[0], null); + assertEquals(200, createBucket.statusCode()); + } + + private String uploadPart(String uploadId, int partNumber, String content) throws Exception + { + byte[] body = content.getBytes(StandardCharsets.UTF_8); + HttpResponse put = send("PUT", + baseUri.resolve(BUCKET + "/" + KEY + "?partNumber=" + partNumber + "&uploadId=" + uploadId), body, null); + assertEquals(200, put.statusCode()); + return put.headers().firstValue("ETag").orElseThrow().replace("\"", ""); + } + + private String extract(String xml, String tag) + { + Matcher matcher = Pattern.compile("<" + tag + ">(.*?)").matcher(xml); + assertTrue(matcher.find(), "missing <" + tag + "> in: " + xml); + return matcher.group(1).replace(""", "").replace("\"", ""); + } + + private HttpResponse send(String method, URI uri, byte[] body, String contentType) throws Exception + { + HttpRequest.Builder builder = HttpRequest.newBuilder(uri) + .method(method, HttpRequest.BodyPublishers.ofByteArray(body)); + if (contentType != null) + { + builder.header("Content-Type", contentType); + } + SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body); + return client.send(builder.build(), HttpResponse.BodyHandlers.discarding()); + } + + private HttpResponse sendForString(String method, URI uri, byte[] body, String contentType) throws Exception + { + HttpRequest.Builder builder = HttpRequest.newBuilder(uri) + .method(method, HttpRequest.BodyPublishers.ofByteArray(body)); + if (contentType != null) + { + builder.header("Content-Type", contentType); + } + SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body); + return client.send(builder.build(), HttpResponse.BodyHandlers.ofString()); + } + + private HttpResponse sendForBytes(String method, URI uri) throws Exception + { + HttpRequest.Builder builder = HttpRequest.newBuilder(uri) + .method(method, HttpRequest.BodyPublishers.ofByteArray(new byte[0])); + SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, new byte[0]); + return client.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray()); + } +}