From 4b069e66bce8f6527eaefe122affbd40511c5c03 Mon Sep 17 00:00:00 2001 From: Andreas Dinauer Date: Thu, 1 Oct 2026 19:39:13 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Init?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pipeline.sh | 1 - .../dinauer/r9t/core/audit/AuditAction.java | 7 + .../dinauer/r9t/core/audit/AuditLogEntry.java | 8 + .../r9t/core/audit/AuditLogJsonMapper.java | 45 ++++++ .../dinauer/r9t/core/audit/AuditLogRepo.java | 146 ++++++++++++++++++ .../r9t/core/audit/AuditLogResource.java | 27 ++++ .../r9t/core/audit/AuditLogService.java | 41 +++++ .../r9t/core/audit/AuditResourceType.java | 7 + .../dinauer/r9t/core/bucket/BucketRepo.java | 77 +++++++++ .../r9t/core/bucket/BucketResource.java | 23 ++- .../r9t/core/bucket/BucketService.java | 34 ++++ .../dinauer/r9t/core/bucket/BucketStats.java | 5 + .../core/bucket/BucketStatsJsonMapper.java | 33 ++++ .../r9t/core/bucket/BucketStatsResource.java | 27 ++++ .../r9t/core/bucket/BucketXmlMapper.java | 13 ++ .../r9t/core/multipart/MultipartUpload.java | 7 +- .../core/multipart/MultipartUploadRepo.java | 11 +- .../multipart/MultipartUploadService.java | 18 ++- .../r9t/core/object/ListObjectsResult.java | 7 + .../dinauer/r9t/core/object/ObjectRepo.java | 145 ++++++++++++++++- .../r9t/core/object/ObjectResource.java | 80 +++++++++- .../r9t/core/object/ObjectService.java | 47 +++++- .../r9t/core/object/ObjectXmlMapper.java | 39 ++++- .../dinauer/r9t/core/s3/FileAttributes.java | 13 ++ .../dev/dinauer/r9t/core/s3/S3ErrorCode.java | 1 + .../java/dev/dinauer/r9t/core/s3/Tag.java | 5 + .../dev/dinauer/r9t/core/s3/TagCodec.java | 54 +++++++ .../dev/dinauer/r9t/core/s3/TagValidator.java | 38 +++++ .../java/dev/dinauer/r9t/core/s3/TagXml.java | 73 +++++++++ .../core/multipart/MultipartUploadTest.java | 39 +++++ .../core/object/ObjectTaggingUploadTest.java | 82 ++++++++++ src/test/resources/application.properties | 2 + 32 files changed, 1131 insertions(+), 24 deletions(-) create mode 100644 src/main/java/dev/dinauer/r9t/core/audit/AuditAction.java create mode 100644 src/main/java/dev/dinauer/r9t/core/audit/AuditLogEntry.java create mode 100644 src/main/java/dev/dinauer/r9t/core/audit/AuditLogJsonMapper.java create mode 100644 src/main/java/dev/dinauer/r9t/core/audit/AuditLogRepo.java create mode 100644 src/main/java/dev/dinauer/r9t/core/audit/AuditLogResource.java create mode 100644 src/main/java/dev/dinauer/r9t/core/audit/AuditLogService.java create mode 100644 src/main/java/dev/dinauer/r9t/core/audit/AuditResourceType.java create mode 100644 src/main/java/dev/dinauer/r9t/core/bucket/BucketStats.java create mode 100644 src/main/java/dev/dinauer/r9t/core/bucket/BucketStatsJsonMapper.java create mode 100644 src/main/java/dev/dinauer/r9t/core/bucket/BucketStatsResource.java create mode 100644 src/main/java/dev/dinauer/r9t/core/object/ListObjectsResult.java create mode 100644 src/main/java/dev/dinauer/r9t/core/s3/Tag.java create mode 100644 src/main/java/dev/dinauer/r9t/core/s3/TagCodec.java create mode 100644 src/main/java/dev/dinauer/r9t/core/s3/TagValidator.java create mode 100644 src/main/java/dev/dinauer/r9t/core/s3/TagXml.java create mode 100644 src/test/java/dev/dinauer/r9t/core/object/ObjectTaggingUploadTest.java diff --git a/pipeline.sh b/pipeline.sh index dc9bcb5..7ae96a4 100755 --- a/pipeline.sh +++ b/pipeline.sh @@ -3,7 +3,6 @@ set -euo pipefail IMAGE="harbor.dinauer.dev/r9t/core" TAG="${1:-1.0.0}" -# cd "$(dirname "${BASH_SOURCE[0]}")" diff --git a/src/main/java/dev/dinauer/r9t/core/audit/AuditAction.java b/src/main/java/dev/dinauer/r9t/core/audit/AuditAction.java new file mode 100644 index 0000000..b2a671b --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/audit/AuditAction.java @@ -0,0 +1,7 @@ +package dev.dinauer.r9t.core.audit; + +public enum AuditAction +{ + CREATE, + DELETE +} diff --git a/src/main/java/dev/dinauer/r9t/core/audit/AuditLogEntry.java b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogEntry.java new file mode 100644 index 0000000..3db17d8 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogEntry.java @@ -0,0 +1,8 @@ +package dev.dinauer.r9t.core.audit; + +import java.time.Instant; + +public record AuditLogEntry(Instant timestamp, AuditAction action, AuditResourceType resourceType, String bucket, + String key, String principal) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/audit/AuditLogJsonMapper.java b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogJsonMapper.java new file mode 100644 index 0000000..fd6f0fd --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogJsonMapper.java @@ -0,0 +1,45 @@ +package dev.dinauer.r9t.core.audit; + +import jakarta.enterprise.context.ApplicationScoped; + +import java.util.List; +import java.util.stream.Collectors; + +@ApplicationScoped +public class AuditLogJsonMapper +{ + public String map(List input) + { + return "[" + input.stream().map(this::map).collect(Collectors.joining(",")) + "]"; + } + + private String map(AuditLogEntry input) + { + return "{" + + "\"timestamp\":\"" + input.timestamp() + "\"," + + "\"action\":\"" + input.action() + "\"," + + "\"resourceType\":\"" + input.resourceType() + "\"," + + "\"bucket\":" + jsonString(input.bucket()) + "," + + "\"key\":" + jsonString(input.key()) + "," + + "\"principal\":" + jsonString(input.principal()) + + "}"; + } + + private String jsonString(String value) + { + if (value == null) + { + return "null"; + } + return "\"" + escape(value) + "\""; + } + + private String escape(String value) + { + return value + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r"); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/audit/AuditLogRepo.java b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogRepo.java new file mode 100644 index 0000000..3bd611b --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogRepo.java @@ -0,0 +1,146 @@ +package dev.dinauer.r9t.core.audit; + +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.S3Exception; +import jakarta.enterprise.context.ApplicationScoped; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.Instant; +import java.util.List; + +@ApplicationScoped +public class AuditLogRepo +{ + @ConfigProperty(name = "dev.dinauer.r9t.workdir", defaultValue = "/var/lib/r9t") + String workdir; + + public void append(AuditLogEntry entry) + { + Path file = logFile(); + try + { + Files.createDirectories(file.getParent()); + Files.writeString(file, toLine(entry) + System.lineSeparator(), StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, "audit-log", e); + } + } + + public List list() + { + Path file = logFile(); + if (!Files.exists(file)) + { + return List.of(); + } + try + { + return Files.readAllLines(file).stream() + .filter(line -> !line.isBlank()) + .map(this::fromLine) + .toList(); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, "audit-log", e); + } + } + + private Path logFile() + { + return Path.of(workdir, ".audit", "audit.log"); + } + + private String toLine(AuditLogEntry entry) + { + return String.join("\t", + entry.timestamp().toString(), + entry.action().name(), + entry.resourceType().name(), + escape(entry.bucket()), + nullableField(entry.key()), + nullableField(entry.principal())); + } + + private AuditLogEntry fromLine(String line) + { + String[] fields = line.split("\t", 6); + return new AuditLogEntry( + Instant.parse(fields[0]), + AuditAction.valueOf(fields[1]), + AuditResourceType.valueOf(fields[2]), + unescape(fields[3]), + parseNullableField(fields[4]), + parseNullableField(fields[5])); + } + + private String nullableField(String value) + { + if (value == null) + { + return ""; + } + return escape(value); + } + + private String parseNullableField(String field) + { + if (field.isEmpty()) + { + return null; + } + return unescape(field); + } + + private String escape(String value) + { + return value + .replace("\\", "\\\\") + .replace("\t", "\\t") + .replace("\n", "\\n") + .replace("\r", "\\r"); + } + + private String unescape(String value) + { + StringBuilder result = new StringBuilder(); + for (int i = 0; i < value.length(); i++) + { + char c = value.charAt(i); + if (c == '\\' && i + 1 < value.length()) + { + i++; + result.append(unescapedChar(value.charAt(i))); + } + else + { + result.append(c); + } + } + return result.toString(); + } + + private char unescapedChar(char c) + { + if (c == 't') + { + return '\t'; + } + if (c == 'n') + { + return '\n'; + } + if (c == 'r') + { + return '\r'; + } + return c; + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/audit/AuditLogResource.java b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogResource.java new file mode 100644 index 0000000..e8d7088 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogResource.java @@ -0,0 +1,27 @@ +package dev.dinauer.r9t.core.audit; + +import io.quarkus.security.Authenticated; +import jakarta.inject.Inject; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +@Path("/_audit-logs") +@Authenticated +public class AuditLogResource +{ + @Inject + AuditLogService auditLogService; + + @Inject + AuditLogJsonMapper auditLogJsonMapper; + + @GET + @Produces(MediaType.APPLICATION_JSON) + public Response list() + { + return Response.ok(auditLogJsonMapper.map(auditLogService.list())).build(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/audit/AuditLogService.java b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogService.java new file mode 100644 index 0000000..58e1624 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogService.java @@ -0,0 +1,41 @@ +package dev.dinauer.r9t.core.audit; + +import io.quarkus.security.identity.SecurityIdentity; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import java.time.Instant; +import java.util.Comparator; +import java.util.List; + +@ApplicationScoped +public class AuditLogService +{ + @Inject + AuditLogRepo auditLogRepo; + + @Inject + SecurityIdentity securityIdentity; + + public void record(AuditAction action, AuditResourceType resourceType, String bucket, String key) + { + AuditLogEntry entry = new AuditLogEntry(Instant.now(), action, resourceType, bucket, key, principal()); + auditLogRepo.append(entry); + } + + public List list() + { + return auditLogRepo.list().stream() + .sorted(Comparator.comparing(AuditLogEntry::timestamp).reversed()) + .toList(); + } + + private String principal() + { + if (securityIdentity.getPrincipal() != null) + { + return securityIdentity.getPrincipal().getName(); + } + return null; + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/audit/AuditResourceType.java b/src/main/java/dev/dinauer/r9t/core/audit/AuditResourceType.java new file mode 100644 index 0000000..aa96f9f --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/audit/AuditResourceType.java @@ -0,0 +1,7 @@ +package dev.dinauer.r9t.core.audit; + +public enum AuditResourceType +{ + BUCKET, + OBJECT +} diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java index dc69cd4..9175fe1 100644 --- a/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java @@ -1,17 +1,23 @@ package dev.dinauer.r9t.core.bucket; +import dev.dinauer.r9t.core.s3.FileAttributes; import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3Exception; +import dev.dinauer.r9t.core.s3.Tag; +import dev.dinauer.r9t.core.s3.TagCodec; import jakarta.annotation.PostConstruct; import jakarta.enterprise.context.ApplicationScoped; import org.eclipse.microprofile.config.inject.ConfigProperty; import java.io.IOException; import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; @ApplicationScoped @@ -97,6 +103,77 @@ public class BucketRepo } } + public List getTags(String bucket) + { + Path target = resolve(bucket); + try + { + if (FileAttributes.exists(target, FileAttributes.TAGS)) + { + return TagCodec.parseQueryString(FileAttributes.read(target, FileAttributes.TAGS)); + } + return List.of(); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); + } + } + + public void putTags(String bucket, List tags) + { + Path target = resolve(bucket); + try + { + FileAttributes.write(target, FileAttributes.TAGS, TagCodec.toQueryString(tags)); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); + } + } + + public void deleteTags(String bucket) + { + Path target = resolve(bucket); + try + { + if (FileAttributes.exists(target, FileAttributes.TAGS)) + { + FileAttributes.delete(target, FileAttributes.TAGS); + } + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); + } + } + + public BucketStats stats(String bucket) + { + Path bucketPath = resolve(bucket); + AtomicLong objectCount = new AtomicLong(); + AtomicLong totalSize = new AtomicLong(); + try + { + Files.walkFileTree(bucketPath, new SimpleFileVisitor<>() + { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) + { + objectCount.incrementAndGet(); + totalSize.addAndGet(attributes.size()); + return FileVisitResult.CONTINUE; + } + }); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); + } + return new BucketStats(bucket, objectCount.get(), totalSize.get()); + } + private Bucket toBucket(Path dir) { try diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketResource.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketResource.java index 4b84c36..718e599 100644 --- a/src/main/java/dev/dinauer/r9t/core/bucket/BucketResource.java +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketResource.java @@ -1,15 +1,22 @@ package dev.dinauer.r9t.core.bucket; +import dev.dinauer.r9t.core.s3.Tag; import io.quarkus.security.Authenticated; import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; import jakarta.ws.rs.DELETE; import jakarta.ws.rs.GET; import jakarta.ws.rs.PUT; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriInfo; + +import java.io.InputStream; +import java.util.List; @Path("/") @Authenticated @@ -30,16 +37,28 @@ public class BucketResource @PUT @Path("{bucket}") - public Response create(@PathParam("bucket") String bucket) + @Consumes(MediaType.WILDCARD) + public Response create(@PathParam("bucket") String bucket, @Context UriInfo uriInfo, InputStream body) { + if (uriInfo.getQueryParameters().containsKey("tagging")) + { + List tags = bucketXmlMapper.mapTagging(body); + bucketService.putTags(bucket, tags); + return Response.ok().build(); + } bucketService.create(bucket); return Response.ok().build(); } @DELETE @Path("{bucket}") - public Response delete(@PathParam("bucket") String bucket) + public Response delete(@PathParam("bucket") String bucket, @Context UriInfo uriInfo) { + if (uriInfo.getQueryParameters().containsKey("tagging")) + { + bucketService.deleteTags(bucket); + return Response.noContent().build(); + } bucketService.delete(bucket); return Response.noContent().build(); } diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketService.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketService.java index 6b46dc8..838bdf6 100644 --- a/src/main/java/dev/dinauer/r9t/core/bucket/BucketService.java +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketService.java @@ -1,7 +1,12 @@ package dev.dinauer.r9t.core.bucket; +import dev.dinauer.r9t.core.audit.AuditAction; +import dev.dinauer.r9t.core.audit.AuditLogService; +import dev.dinauer.r9t.core.audit.AuditResourceType; import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3Exception; +import dev.dinauer.r9t.core.s3.Tag; +import dev.dinauer.r9t.core.s3.TagValidator; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -16,11 +21,19 @@ public class BucketService @Inject BucketRepo bucketRepo; + @Inject + AuditLogService auditLogService; + public List list() { return bucketRepo.list(); } + public List stats() + { + return bucketRepo.list().stream().map(bucket -> bucketRepo.stats(bucket.name())).toList(); + } + public boolean exists(String name) { return bucketRepo.exists(name); @@ -34,12 +47,14 @@ public class BucketService throw new S3Exception(S3ErrorCode.BUCKET_ALREADY_EXISTS, name); } bucketRepo.create(name); + auditLogService.record(AuditAction.CREATE, AuditResourceType.BUCKET, name, null); } public void delete(String name) { require(name); bucketRepo.delete(name); + auditLogService.record(AuditAction.DELETE, AuditResourceType.BUCKET, name, null); } public void require(String name) @@ -50,6 +65,25 @@ public class BucketService } } + public List getTags(String name) + { + require(name); + return bucketRepo.getTags(name); + } + + public void putTags(String name, List tags) + { + require(name); + TagValidator.validate(tags); + bucketRepo.putTags(name, tags); + } + + public void deleteTags(String name) + { + require(name); + bucketRepo.deleteTags(name); + } + private void validate(String name) { if (!NAME_PATTERN.matcher(name).matches()) diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketStats.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketStats.java new file mode 100644 index 0000000..5a80070 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketStats.java @@ -0,0 +1,5 @@ +package dev.dinauer.r9t.core.bucket; + +public record BucketStats(String name, long objectCount, long totalSize) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketStatsJsonMapper.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketStatsJsonMapper.java new file mode 100644 index 0000000..e6e670d --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketStatsJsonMapper.java @@ -0,0 +1,33 @@ +package dev.dinauer.r9t.core.bucket; + +import jakarta.enterprise.context.ApplicationScoped; + +import java.util.List; +import java.util.stream.Collectors; + +@ApplicationScoped +public class BucketStatsJsonMapper +{ + public String map(List input) + { + return "[" + input.stream().map(this::map).collect(Collectors.joining(",")) + "]"; + } + + private String map(BucketStats input) + { + return "{" + + "\"name\":\"" + escape(input.name()) + "\"," + + "\"objectCount\":" + input.objectCount() + "," + + "\"totalSize\":" + input.totalSize() + + "}"; + } + + private String escape(String value) + { + return value + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r"); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketStatsResource.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketStatsResource.java new file mode 100644 index 0000000..49ca115 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketStatsResource.java @@ -0,0 +1,27 @@ +package dev.dinauer.r9t.core.bucket; + +import io.quarkus.security.Authenticated; +import jakarta.inject.Inject; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +@Path("/_bucket-stats") +@Authenticated +public class BucketStatsResource +{ + @Inject + BucketService bucketService; + + @Inject + BucketStatsJsonMapper bucketStatsJsonMapper; + + @GET + @Produces(MediaType.APPLICATION_JSON) + public Response list() + { + return Response.ok(bucketStatsJsonMapper.map(bucketService.stats())).build(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketXmlMapper.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketXmlMapper.java index ca8ef8a..7467d5b 100644 --- a/src/main/java/dev/dinauer/r9t/core/bucket/BucketXmlMapper.java +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketXmlMapper.java @@ -1,8 +1,11 @@ package dev.dinauer.r9t.core.bucket; import dev.dinauer.r9t.core.s3.S3Xml; +import dev.dinauer.r9t.core.s3.Tag; +import dev.dinauer.r9t.core.s3.TagXml; import jakarta.enterprise.context.ApplicationScoped; +import java.io.InputStream; import java.util.List; import java.util.stream.Collectors; @@ -27,4 +30,14 @@ public class BucketXmlMapper + " \n" + ""; } + + public String mapTagging(List tags) + { + return TagXml.map(tags); + } + + public List mapTagging(InputStream body) + { + return TagXml.parse(body); + } } diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUpload.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUpload.java index 16bf065..eb33fe8 100644 --- a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUpload.java +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUpload.java @@ -1,7 +1,10 @@ package dev.dinauer.r9t.core.multipart; -import java.time.Instant; +import dev.dinauer.r9t.core.s3.Tag; -public record MultipartUpload(String uploadId, String bucket, String key, String contentType, Instant initiated) +import java.time.Instant; +import java.util.List; + +public record MultipartUpload(String uploadId, String bucket, String key, String contentType, Instant initiated, List tags) { } diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java index fbda3da..aaffc11 100644 --- a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java @@ -5,6 +5,8 @@ import dev.dinauer.r9t.core.s3.DigestCopy; import dev.dinauer.r9t.core.s3.FileAttributes; import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3Exception; +import dev.dinauer.r9t.core.s3.Tag; +import dev.dinauer.r9t.core.s3.TagCodec; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -14,6 +16,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; import java.util.Comparator; +import java.util.List; import java.util.UUID; import java.util.stream.Stream; @@ -43,7 +46,7 @@ public class MultipartUploadRepo return Files.isDirectory(resolve(uploadId)); } - public MultipartUpload create(String bucket, String key, String contentType) + public MultipartUpload create(String bucket, String key, String contentType, List tags) { String uploadId = UUID.randomUUID().toString().replace("-", ""); Path dir = resolve(uploadId); @@ -55,12 +58,13 @@ public class MultipartUploadRepo FileAttributes.write(dir, KEY_ATTRIBUTE, key); FileAttributes.write(dir, FileAttributes.CONTENT_TYPE, contentType); FileAttributes.write(dir, INITIATED_ATTRIBUTE, initiated.toString()); + FileAttributes.write(dir, FileAttributes.TAGS, TagCodec.toQueryString(tags)); } catch (IOException e) { throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); } - return new MultipartUpload(uploadId, bucket, key, contentType, initiated); + return new MultipartUpload(uploadId, bucket, key, contentType, initiated, tags); } public MultipartUpload metadata(String uploadId) @@ -72,7 +76,8 @@ public class MultipartUploadRepo String key = FileAttributes.read(dir, KEY_ATTRIBUTE); String contentType = FileAttributes.read(dir, FileAttributes.CONTENT_TYPE); Instant initiated = Instant.parse(FileAttributes.read(dir, INITIATED_ATTRIBUTE)); - return new MultipartUpload(uploadId, bucket, key, contentType, initiated); + List tags = TagCodec.parseQueryString(FileAttributes.read(dir, FileAttributes.TAGS)); + return new MultipartUpload(uploadId, bucket, key, contentType, initiated, tags); } catch (IOException e) { diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java index f540ee5..471a8f2 100644 --- a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java @@ -1,11 +1,16 @@ package dev.dinauer.r9t.core.multipart; +import dev.dinauer.r9t.core.audit.AuditAction; +import dev.dinauer.r9t.core.audit.AuditLogService; +import dev.dinauer.r9t.core.audit.AuditResourceType; import dev.dinauer.r9t.core.bucket.BucketService; import dev.dinauer.r9t.core.object.ObjectRepo; import dev.dinauer.r9t.core.object.S3Object; import dev.dinauer.r9t.core.s3.DigestCopy; import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3Exception; +import dev.dinauer.r9t.core.s3.Tag; +import dev.dinauer.r9t.core.s3.TagValidator; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -34,7 +39,10 @@ public class MultipartUploadService @Inject BucketService bucketService; - public MultipartUpload initiate(String bucket, String key, String contentType) + @Inject + AuditLogService auditLogService; + + public MultipartUpload initiate(String bucket, String key, String contentType, List tags) { bucketService.require(bucket); String resolvedContentType = contentType; @@ -42,7 +50,8 @@ public class MultipartUploadService { resolvedContentType = "application/octet-stream"; } - return multipartUploadRepo.create(bucket, key, resolvedContentType); + TagValidator.validate(tags); + return multipartUploadRepo.create(bucket, key, resolvedContentType, tags); } public String uploadPart(String bucket, String key, String uploadId, Integer partNumber, InputStream data) @@ -72,7 +81,12 @@ public class MultipartUploadService { throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); } + if (!upload.tags().isEmpty()) + { + objectRepo.putTags(bucket, key, upload.tags()); + } multipartUploadRepo.delete(uploadId); + auditLogService.record(AuditAction.CREATE, AuditResourceType.OBJECT, bucket, key); return objectRepo.metadata(bucket, key); } diff --git a/src/main/java/dev/dinauer/r9t/core/object/ListObjectsResult.java b/src/main/java/dev/dinauer/r9t/core/object/ListObjectsResult.java new file mode 100644 index 0000000..de76dd8 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/object/ListObjectsResult.java @@ -0,0 +1,7 @@ +package dev.dinauer.r9t.core.object; + +import java.util.List; + +public record ListObjectsResult(List contents, List commonPrefixes) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java index e14eb4c..91d4944 100644 --- a/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java @@ -5,6 +5,8 @@ import dev.dinauer.r9t.core.s3.DigestCopy; import dev.dinauer.r9t.core.s3.FileAttributes; import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3Exception; +import dev.dinauer.r9t.core.s3.Tag; +import dev.dinauer.r9t.core.s3.TagCodec; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; @@ -14,7 +16,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.stream.Stream; @ApplicationScoped @@ -48,6 +53,7 @@ public class ObjectRepo String etag = DigestCopy.copyAndHashMd5(data, target); FileAttributes.write(target, FileAttributes.ETAG, etag); FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType); + clearTags(target); return etag; } catch (IOException e) @@ -65,6 +71,7 @@ public class ObjectRepo Files.copy(data, target, StandardCopyOption.REPLACE_EXISTING); FileAttributes.write(target, FileAttributes.ETAG, etag); FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType); + clearTags(target); } catch (IOException e) { @@ -72,6 +79,57 @@ public class ObjectRepo } } + public List getTags(String bucket, String key) + { + Path target = resolve(bucket, key); + try + { + if (FileAttributes.exists(target, FileAttributes.TAGS)) + { + return TagCodec.parseQueryString(FileAttributes.read(target, FileAttributes.TAGS)); + } + return List.of(); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); + } + } + + public void putTags(String bucket, String key, List tags) + { + Path target = resolve(bucket, key); + try + { + FileAttributes.write(target, FileAttributes.TAGS, TagCodec.toQueryString(tags)); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); + } + } + + public void deleteTags(String bucket, String key) + { + Path target = resolve(bucket, key); + try + { + clearTags(target); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); + } + } + + private void clearTags(Path target) throws IOException + { + if (FileAttributes.exists(target, FileAttributes.TAGS)) + { + FileAttributes.delete(target, FileAttributes.TAGS); + } + } + public S3Object metadata(String bucket, String key) { Path target = resolve(bucket, key); @@ -105,14 +163,17 @@ public class ObjectRepo } } - public List list(String bucket) + public List searchByTags(String bucket, Map tagFilters) { Path bucketPath = bucketRepo.resolve(bucket); try (Stream entries = Files.walk(bucketPath)) { return entries .filter(Files::isRegularFile) - .map(path -> metadata(bucket, bucketPath.relativize(path).toString())) + .map(path -> bucketPath.relativize(path).toString()) + .filter(key -> matchesAllTags(bucket, key, tagFilters)) + .sorted() + .map(key -> metadata(bucket, key)) .toList(); } catch (IOException e) @@ -120,4 +181,84 @@ public class ObjectRepo throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); } } + + private boolean matchesAllTags(String bucket, String key, Map tagFilters) + { + List tags = getTags(bucket, key); + for (Map.Entry filter : tagFilters.entrySet()) + { + if (!matchesTag(tags, filter.getKey(), filter.getValue())) + { + return false; + } + } + return true; + } + + private boolean matchesTag(List tags, String key, String value) + { + return tags.stream().anyMatch(tag -> tag.key().equals(key) && tag.value().equals(value)); + } + + public ListObjectsResult list(String bucket, String prefix, String delimiter) + { + Path bucketPath = bucketRepo.resolve(bucket); + try (Stream entries = Files.walk(bucketPath)) + { + List keys = entries + .filter(Files::isRegularFile) + .map(path -> bucketPath.relativize(path).toString()) + .filter(key -> matchesPrefix(key, prefix)) + .sorted() + .toList(); + return group(bucket, keys, normalizedPrefix(prefix), delimiter); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); + } + } + + private boolean matchesPrefix(String key, String prefix) + { + if (prefix == null) + { + return true; + } + return key.startsWith(prefix); + } + + private String normalizedPrefix(String prefix) + { + if (prefix == null) + { + return ""; + } + return prefix; + } + + private ListObjectsResult group(String bucket, List keys, String prefix, String delimiter) + { + if (delimiter == null) + { + List contents = keys.stream().map(key -> metadata(bucket, key)).toList(); + return new ListObjectsResult(contents, List.of()); + } + List contents = new ArrayList<>(); + LinkedHashSet commonPrefixes = new LinkedHashSet<>(); + for (String key : keys) + { + String rest = key.substring(prefix.length()); + int delimiterIndex = rest.indexOf(delimiter); + if (delimiterIndex == -1) + { + contents.add(metadata(bucket, key)); + } + else + { + commonPrefixes.add(prefix + rest.substring(0, delimiterIndex + delimiter.length())); + } + } + return new ListObjectsResult(contents, List.copyOf(commonPrefixes)); + } } diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java index d1eaaeb..c7db1d2 100644 --- a/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java @@ -1,10 +1,14 @@ package dev.dinauer.r9t.core.object; import dev.dinauer.r9t.core.auth.AwsChunkedInputStream; +import dev.dinauer.r9t.core.bucket.BucketService; +import dev.dinauer.r9t.core.bucket.BucketXmlMapper; import dev.dinauer.r9t.core.multipart.CompletedPart; import dev.dinauer.r9t.core.multipart.MultipartUpload; import dev.dinauer.r9t.core.multipart.MultipartUploadService; import dev.dinauer.r9t.core.multipart.MultipartUploadXmlMapper; +import dev.dinauer.r9t.core.s3.Tag; +import dev.dinauer.r9t.core.s3.TagCodec; import io.quarkus.security.Authenticated; import org.apache.commons.lang3.StringUtils; import jakarta.inject.Inject; @@ -26,12 +30,16 @@ import jakarta.ws.rs.core.UriInfo; import java.io.InputStream; import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; @Path("/{bucket}") @Authenticated public class ObjectResource { + private static final String TAG_QUERY_PREFIX = "x-tag-"; + @Inject ObjectService objectService; @@ -44,11 +52,45 @@ public class ObjectResource @Inject MultipartUploadXmlMapper multipartUploadXmlMapper; + @Inject + BucketService bucketService; + + @Inject + BucketXmlMapper bucketXmlMapper; + @GET @Produces(MediaType.APPLICATION_XML) - public Response list(@PathParam("bucket") String bucket) + public Response list(@PathParam("bucket") String bucket, @QueryParam("prefix") String prefix, + @QueryParam("delimiter") String delimiter, @Context UriInfo uriInfo) { - return Response.ok(objectXmlMapper.map(bucket, objectService.list(bucket))).build(); + if (uriInfo.getQueryParameters().containsKey("tagging")) + { + List tags = bucketService.getTags(bucket); + return Response.ok(bucketXmlMapper.mapTagging(tags)).build(); + } + Map tagFilters = tagFilters(uriInfo); + if (!tagFilters.isEmpty()) + { + List matches = objectService.searchByTags(bucket, tagFilters); + ListObjectsResult result = new ListObjectsResult(matches, List.of()); + return Response.ok(objectXmlMapper.map(bucket, null, null, result)).build(); + } + ListObjectsResult result = objectService.list(bucket, prefix, delimiter); + return Response.ok(objectXmlMapper.map(bucket, prefix, delimiter, result)).build(); + } + + private Map tagFilters(UriInfo uriInfo) + { + Map filters = new LinkedHashMap<>(); + for (Map.Entry> param : uriInfo.getQueryParameters().entrySet()) + { + if (param.getKey().startsWith(TAG_QUERY_PREFIX)) + { + String tagName = param.getKey().substring(TAG_QUERY_PREFIX.length()); + filters.put(tagName, param.getValue().get(0)); + } + } + return filters; } @PUT @@ -56,9 +98,17 @@ public class ObjectResource @Consumes(MediaType.WILDCARD) public Response put(@PathParam("bucket") String bucket, @PathParam("key") String key, @QueryParam("uploadId") String uploadId, @QueryParam("partNumber") Integer partNumber, + @Context UriInfo uriInfo, @HeaderParam("Content-Type") String contentType, - @HeaderParam("X-Amz-Content-Sha256") String contentSha256, InputStream body) + @HeaderParam("X-Amz-Content-Sha256") String contentSha256, + @HeaderParam("x-amz-tagging") String tagging, InputStream body) { + if (uriInfo.getQueryParameters().containsKey("tagging")) + { + List tags = objectXmlMapper.mapTagging(body); + objectService.putTags(bucket, key, tags); + return Response.ok().build(); + } InputStream payload = body; if (StringUtils.startsWith(contentSha256, "STREAMING-")) { @@ -70,6 +120,10 @@ public class ObjectResource return Response.ok().header("ETag", "\"" + partEtag + "\"").build(); } String etag = objectService.put(bucket, key, contentType, payload); + if (tagging != null) + { + objectService.putTags(bucket, key, TagCodec.parseQueryString(tagging)); + } return Response.ok().header("ETag", "\"" + etag + "\"").build(); } @@ -79,11 +133,13 @@ public class ObjectResource @Produces(MediaType.APPLICATION_XML) public Response post(@PathParam("bucket") String bucket, @PathParam("key") String key, @Context UriInfo uriInfo, @QueryParam("uploadId") String uploadId, - @HeaderParam("Content-Type") String contentType, InputStream body) + @HeaderParam("Content-Type") String contentType, + @HeaderParam("x-amz-tagging") String tagging, InputStream body) { if (uriInfo.getQueryParameters().containsKey("uploads")) { - MultipartUpload upload = multipartUploadService.initiate(bucket, key, contentType); + List tags = TagCodec.parseQueryString(tagging); + MultipartUpload upload = multipartUploadService.initiate(bucket, key, contentType, tags); return Response.ok(multipartUploadXmlMapper.map(upload)).build(); } List parts = multipartUploadXmlMapper.map(body); @@ -93,8 +149,13 @@ public class ObjectResource @GET @Path("{key:.+}") - public Response get(@PathParam("bucket") String bucket, @PathParam("key") String key) + public Response get(@PathParam("bucket") String bucket, @PathParam("key") String key, @Context UriInfo uriInfo) { + if (uriInfo.getQueryParameters().containsKey("tagging")) + { + List tags = objectService.getTags(bucket, key); + return Response.ok(objectXmlMapper.mapTagging(tags)).type(MediaType.APPLICATION_XML).build(); + } S3Object metadata = objectService.metadata(bucket, key); return Response.ok(objectService.file(bucket, key).toFile()) .type(metadata.contentType()) @@ -119,8 +180,13 @@ public class ObjectResource @DELETE @Path("{key:.+}") public Response delete(@PathParam("bucket") String bucket, @PathParam("key") String key, - @QueryParam("uploadId") String uploadId) + @QueryParam("uploadId") String uploadId, @Context UriInfo uriInfo) { + if (uriInfo.getQueryParameters().containsKey("tagging")) + { + objectService.deleteTags(bucket, key); + return Response.noContent().build(); + } if (uploadId != null) { multipartUploadService.abort(bucket, key, uploadId); diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectService.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectService.java index a17f165..2855187 100644 --- a/src/main/java/dev/dinauer/r9t/core/object/ObjectService.java +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectService.java @@ -1,14 +1,20 @@ package dev.dinauer.r9t.core.object; +import dev.dinauer.r9t.core.audit.AuditAction; +import dev.dinauer.r9t.core.audit.AuditLogService; +import dev.dinauer.r9t.core.audit.AuditResourceType; import dev.dinauer.r9t.core.bucket.BucketService; import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3Exception; +import dev.dinauer.r9t.core.s3.Tag; +import dev.dinauer.r9t.core.s3.TagValidator; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.io.InputStream; import java.nio.file.Path; import java.util.List; +import java.util.Map; @ApplicationScoped public class ObjectService @@ -19,10 +25,23 @@ public class ObjectService @Inject BucketService bucketService; - public List list(String bucket) + @Inject + AuditLogService auditLogService; + + public ListObjectsResult list(String bucket, String prefix, String delimiter) { bucketService.require(bucket); - return objectRepo.list(bucket); + return objectRepo.list(bucket, prefix, delimiter); + } + + public List searchByTags(String bucket, Map tagFilters) + { + bucketService.require(bucket); + if (tagFilters.isEmpty()) + { + throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, "x-tag-*"); + } + return objectRepo.searchByTags(bucket, tagFilters); } public String put(String bucket, String key, String contentType, InputStream data) @@ -33,7 +52,9 @@ public class ObjectService { resolvedContentType = "application/octet-stream"; } - return objectRepo.put(bucket, key, resolvedContentType, data); + String etag = objectRepo.put(bucket, key, resolvedContentType, data); + auditLogService.record(AuditAction.CREATE, AuditResourceType.OBJECT, bucket, key); + return etag; } public S3Object metadata(String bucket, String key) @@ -56,5 +77,25 @@ public class ObjectService { bucketService.require(bucket); objectRepo.delete(bucket, key); + auditLogService.record(AuditAction.DELETE, AuditResourceType.OBJECT, bucket, key); + } + + public List getTags(String bucket, String key) + { + metadata(bucket, key); + return objectRepo.getTags(bucket, key); + } + + public void putTags(String bucket, String key, List tags) + { + metadata(bucket, key); + TagValidator.validate(tags); + objectRepo.putTags(bucket, key, tags); + } + + public void deleteTags(String bucket, String key) + { + metadata(bucket, key); + objectRepo.deleteTags(bucket, key); } } diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectXmlMapper.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectXmlMapper.java index 0fd2d29..70d71c5 100644 --- a/src/main/java/dev/dinauer/r9t/core/object/ObjectXmlMapper.java +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectXmlMapper.java @@ -1,8 +1,11 @@ package dev.dinauer.r9t.core.object; import dev.dinauer.r9t.core.s3.S3Xml; +import dev.dinauer.r9t.core.s3.Tag; +import dev.dinauer.r9t.core.s3.TagXml; import jakarta.enterprise.context.ApplicationScoped; +import java.io.InputStream; import java.util.List; import java.util.stream.Collectors; @@ -19,14 +22,46 @@ public class ObjectXmlMapper + ""; } - public String map(String bucket, List input) + public String map(String bucket, String prefix, String delimiter, ListObjectsResult input) { - String contents = input.stream().map(this::map).collect(Collectors.joining()); + String contents = input.contents().stream().map(this::map).collect(Collectors.joining()); + String commonPrefixes = input.commonPrefixes().stream().map(this::mapCommonPrefix).collect(Collectors.joining()); + String prefixTag = ""; + if (prefix != null) + { + prefixTag = " " + S3Xml.escape(prefix) + "\n"; + } + String delimiterTag = ""; + if (delimiter != null) + { + delimiterTag = " " + S3Xml.escape(delimiter) + "\n"; + } return "\n" + "\n" + " " + S3Xml.escape(bucket) + "\n" + + prefixTag + + delimiterTag + + " " + input.contents().size() + "\n" + " false\n" + contents + + commonPrefixes + ""; } + + public String mapTagging(List tags) + { + return TagXml.map(tags); + } + + public List mapTagging(InputStream body) + { + return TagXml.parse(body); + } + + private String mapCommonPrefix(String prefix) + { + return "\n" + + " " + S3Xml.escape(prefix) + "\n" + + ""; + } } diff --git a/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java b/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java index 9af5b59..51e1428 100644 --- a/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java +++ b/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java @@ -11,6 +11,7 @@ public final class FileAttributes { public static final String ETAG = "user.etag"; public static final String CONTENT_TYPE = "user.contentType"; + public static final String TAGS = "user.tags"; private FileAttributes() { @@ -30,4 +31,16 @@ public final class FileAttributes buffer.flip(); return StandardCharsets.UTF_8.decode(buffer).toString(); } + + public static boolean exists(Path file, String name) throws IOException + { + UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); + return view.list().contains(name); + } + + public static void delete(Path file, String name) throws IOException + { + UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); + view.delete(name); + } } diff --git a/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java b/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java index 75f5616..e60d03b 100644 --- a/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java +++ b/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java @@ -11,6 +11,7 @@ public enum S3ErrorCode NO_SUCH_UPLOAD("NoSuchUpload", 404, "The specified multipart upload does not exist"), INVALID_PART("InvalidPart", 400, "One or more of the specified parts could not be found"), MALFORMED_XML("MalformedXML", 400, "The XML you provided was not well-formed or did not validate against our published schema"), + INVALID_TAG("InvalidTag", 400, "The tag provided was not a valid tag. This error can occur if the tag did not pass input validation."), INTERNAL_ERROR("InternalError", 500, "We encountered an internal error"); private final String code; diff --git a/src/main/java/dev/dinauer/r9t/core/s3/Tag.java b/src/main/java/dev/dinauer/r9t/core/s3/Tag.java new file mode 100644 index 0000000..bf4488f --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/s3/Tag.java @@ -0,0 +1,5 @@ +package dev.dinauer.r9t.core.s3; + +public record Tag(String key, String value) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/s3/TagCodec.java b/src/main/java/dev/dinauer/r9t/core/s3/TagCodec.java new file mode 100644 index 0000000..003257a --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/s3/TagCodec.java @@ -0,0 +1,54 @@ +package dev.dinauer.r9t.core.s3; + +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +public final class TagCodec +{ + private TagCodec() + { + } + + public static List parseQueryString(String value) + { + List tags = new ArrayList<>(); + if (value != null && !value.isBlank()) + { + for (String pair : value.split("&")) + { + tags.add(toTag(pair)); + } + } + return tags; + } + + public static String toQueryString(List tags) + { + List pairs = new ArrayList<>(); + for (Tag tag : tags) + { + pairs.add(encode(tag.key()) + "=" + encode(tag.value())); + } + return String.join("&", pairs); + } + + private static Tag toTag(String pair) + { + String[] parts = pair.split("=", 2); + String key = URLDecoder.decode(parts[0], StandardCharsets.UTF_8); + String value = ""; + if (parts.length > 1) + { + value = URLDecoder.decode(parts[1], StandardCharsets.UTF_8); + } + return new Tag(key, value); + } + + private static String encode(String value) + { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/s3/TagValidator.java b/src/main/java/dev/dinauer/r9t/core/s3/TagValidator.java new file mode 100644 index 0000000..55b9250 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/s3/TagValidator.java @@ -0,0 +1,38 @@ +package dev.dinauer.r9t.core.s3; + +import java.util.List; + +public final class TagValidator +{ + private static final int MAX_TAGS = 10; + private static final int MAX_KEY_LENGTH = 128; + private static final int MAX_VALUE_LENGTH = 256; + + private TagValidator() + { + } + + public static void validate(List tags) + { + if (tags.size() > MAX_TAGS) + { + throw new S3Exception(S3ErrorCode.INVALID_TAG, "TagSet"); + } + for (Tag tag : tags) + { + validate(tag); + } + } + + private static void validate(Tag tag) + { + if (tag.key() == null || tag.key().isEmpty() || tag.key().length() > MAX_KEY_LENGTH) + { + throw new S3Exception(S3ErrorCode.INVALID_TAG, String.valueOf(tag.key())); + } + if (tag.value() == null || tag.value().length() > MAX_VALUE_LENGTH) + { + throw new S3Exception(S3ErrorCode.INVALID_TAG, tag.key()); + } + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/s3/TagXml.java b/src/main/java/dev/dinauer/r9t/core/s3/TagXml.java new file mode 100644 index 0000000..6ca901e --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/s3/TagXml.java @@ -0,0 +1,73 @@ +package dev.dinauer.r9t.core.s3; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +public final class TagXml +{ + private TagXml() + { + } + + public static String map(List tags) + { + StringBuilder tagSet = new StringBuilder(); + for (Tag tag : tags) + { + tagSet.append(" \n") + .append(" ").append(S3Xml.escape(tag.key())).append("\n") + .append(" ").append(S3Xml.escape(tag.value())).append("\n") + .append(" \n"); + } + return "\n" + + "\n" + + " \n" + + tagSet + + " \n" + + ""; + } + + public static List parse(InputStream body) + { + try + { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(body); + NodeList tagNodes = document.getElementsByTagName("Tag"); + List tags = new ArrayList<>(); + for (int i = 0; i < tagNodes.getLength(); i++) + { + tags.add(toTag((Element) tagNodes.item(i))); + } + return tags; + } + catch (ParserConfigurationException | SAXException | IOException | NullPointerException e) + { + throw new S3Exception(S3ErrorCode.MALFORMED_XML, "Tagging", e); + } + } + + private static Tag toTag(Element tagElement) + { + String key = text(tagElement, "Key"); + String value = text(tagElement, "Value"); + return new Tag(key, value); + } + + private static String text(Element parent, String tagName) + { + return parent.getElementsByTagName(tagName).item(0).getTextContent(); + } +} diff --git a/src/test/java/dev/dinauer/r9t/core/multipart/MultipartUploadTest.java b/src/test/java/dev/dinauer/r9t/core/multipart/MultipartUploadTest.java index 3803ad3..869960e 100644 --- a/src/test/java/dev/dinauer/r9t/core/multipart/MultipartUploadTest.java +++ b/src/test/java/dev/dinauer/r9t/core/multipart/MultipartUploadTest.java @@ -59,6 +59,31 @@ public class MultipartUploadTest assertEquals("hello world", new String(get.body(), StandardCharsets.UTF_8)); } + @Test + public void initiateWithTaggingAppliesTagsToCompletedObject() throws Exception + { + createBucket(); + + HttpResponse initiate = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploads="), + new byte[0], "text/plain", "x-amz-tagging", "job=nightly-backup&env=staging"); + assertEquals(200, initiate.statusCode()); + String uploadId = extract(initiate.body(), "UploadId"); + + String etag = uploadPart(uploadId, 1, "hello world"); + + String completeBody = "" + + "1\"" + etag + "\"" + + ""; + HttpResponse complete = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploadId=" + uploadId), + completeBody.getBytes(StandardCharsets.UTF_8), null); + assertEquals(200, complete.statusCode()); + + HttpResponse tagging = sendForString("GET", baseUri.resolve(BUCKET + "/" + KEY + "?tagging="), new byte[0], null); + assertEquals(200, tagging.statusCode()); + assertTrue(tagging.body().contains("job\n nightly-backup"), + "missing tag in tagging response: " + tagging.body()); + } + @Test public void abortDiscardsStagedParts() throws Exception { @@ -132,6 +157,20 @@ public class MultipartUploadTest return client.send(builder.build(), HttpResponse.BodyHandlers.ofString()); } + private HttpResponse sendForString(String method, URI uri, byte[] body, String contentType, + String headerName, String headerValue) throws Exception + { + HttpRequest.Builder builder = HttpRequest.newBuilder(uri) + .method(method, HttpRequest.BodyPublishers.ofByteArray(body)) + .header(headerName, headerValue); + if (contentType != null) + { + builder.header("Content-Type", contentType); + } + SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body); + return client.send(builder.build(), HttpResponse.BodyHandlers.ofString()); + } + private HttpResponse sendForBytes(String method, URI uri) throws Exception { HttpRequest.Builder builder = HttpRequest.newBuilder(uri) diff --git a/src/test/java/dev/dinauer/r9t/core/object/ObjectTaggingUploadTest.java b/src/test/java/dev/dinauer/r9t/core/object/ObjectTaggingUploadTest.java new file mode 100644 index 0000000..0a151a0 --- /dev/null +++ b/src/test/java/dev/dinauer/r9t/core/object/ObjectTaggingUploadTest.java @@ -0,0 +1,82 @@ +package dev.dinauer.r9t.core.object; + +import dev.dinauer.r9t.core.auth.SigV4TestSigner; +import io.quarkus.test.common.http.TestHTTPResource; +import io.quarkus.test.junit.QuarkusTest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@QuarkusTest +public class ObjectTaggingUploadTest +{ + private static final String BUCKET = "tagging-upload-test-bucket"; + private static final String KEY = "tagging-upload-test.txt"; + private static final String ACCESS_KEY = "testkey"; + private static final String SECRET_KEY = "testsecret"; + + @TestHTTPResource + URI baseUri; + + private final HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); + + @Test + public void uploadWithTagging() throws Exception + { + HttpResponse createBucket = send("PUT", baseUri.resolve(BUCKET), new byte[0], null, null); + assertEquals(200, createBucket.statusCode()); + + byte[] content = "hello tagging test".getBytes(StandardCharsets.UTF_8); + HttpResponse put = send("PUT", baseUri.resolve(BUCKET + "/" + KEY), content, "text/plain", + "project=r9t&owner=andreas"); + assertEquals(200, put.statusCode()); + assertTrue(put.headers().firstValue("ETag").isPresent()); + + HttpResponse tagging = sendForBytes("GET", baseUri.resolve(BUCKET + "/" + KEY + "?tagging=")); + assertEquals(200, tagging.statusCode()); + String xml = new String(tagging.body(), StandardCharsets.UTF_8); + assertTrue(xml.contains("project")); + assertTrue(xml.contains("r9t")); + assertTrue(xml.contains("owner")); + assertTrue(xml.contains("andreas")); + } + + @AfterEach + public void cleanup() throws Exception + { + send("DELETE", baseUri.resolve(BUCKET + "/" + KEY), new byte[0], null, null); + send("DELETE", baseUri.resolve(BUCKET), new byte[0], null, null); + } + + private HttpResponse send(String method, URI uri, byte[] body, String contentType, String tagging) throws Exception + { + HttpRequest.Builder builder = HttpRequest.newBuilder(uri) + .method(method, HttpRequest.BodyPublishers.ofByteArray(body)); + if (contentType != null) + { + builder.header("Content-Type", contentType); + } + if (tagging != null) + { + builder.header("x-amz-tagging", tagging); + } + SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body); + return client.send(builder.build(), HttpResponse.BodyHandlers.discarding()); + } + + private HttpResponse sendForBytes(String method, URI uri) throws Exception + { + HttpRequest.Builder builder = HttpRequest.newBuilder(uri) + .method(method, HttpRequest.BodyPublishers.ofByteArray(new byte[0])); + SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, new byte[0]); + return client.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray()); + } +} diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 29ecf01..129f0bd 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -1,3 +1,5 @@ +config_ordinal=1000 + dev.dinauer.r9t.workdir=target/test-r9t dev.dinauer.r9t.credentials.access-key=testkey dev.dinauer.r9t.credentials.secret-key=testsecret