🎉 Init

This commit is contained in:
2026-10-01 19:39:13 +02:00
parent 0b8e36fe58
commit 4b069e66bc
32 changed files with 1131 additions and 24 deletions
-1
View File
@@ -3,7 +3,6 @@ set -euo pipefail
IMAGE="harbor.dinauer.dev/r9t/core" IMAGE="harbor.dinauer.dev/r9t/core"
TAG="${1:-1.0.0}" TAG="${1:-1.0.0}"
#
cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(dirname "${BASH_SOURCE[0]}")"
@@ -0,0 +1,7 @@
package dev.dinauer.r9t.core.audit;
public enum AuditAction
{
CREATE,
DELETE
}
@@ -0,0 +1,8 @@
package dev.dinauer.r9t.core.audit;
import java.time.Instant;
public record AuditLogEntry(Instant timestamp, AuditAction action, AuditResourceType resourceType, String bucket,
String key, String principal)
{
}
@@ -0,0 +1,45 @@
package dev.dinauer.r9t.core.audit;
import jakarta.enterprise.context.ApplicationScoped;
import java.util.List;
import java.util.stream.Collectors;
@ApplicationScoped
public class AuditLogJsonMapper
{
public String map(List<AuditLogEntry> input)
{
return "[" + input.stream().map(this::map).collect(Collectors.joining(",")) + "]";
}
private String map(AuditLogEntry input)
{
return "{"
+ "\"timestamp\":\"" + input.timestamp() + "\","
+ "\"action\":\"" + input.action() + "\","
+ "\"resourceType\":\"" + input.resourceType() + "\","
+ "\"bucket\":" + jsonString(input.bucket()) + ","
+ "\"key\":" + jsonString(input.key()) + ","
+ "\"principal\":" + jsonString(input.principal())
+ "}";
}
private String jsonString(String value)
{
if (value == null)
{
return "null";
}
return "\"" + escape(value) + "\"";
}
private String escape(String value)
{
return value
.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r");
}
}
@@ -0,0 +1,146 @@
package dev.dinauer.r9t.core.audit;
import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception;
import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.time.Instant;
import java.util.List;
@ApplicationScoped
public class AuditLogRepo
{
@ConfigProperty(name = "dev.dinauer.r9t.workdir", defaultValue = "/var/lib/r9t")
String workdir;
public void append(AuditLogEntry entry)
{
Path file = logFile();
try
{
Files.createDirectories(file.getParent());
Files.writeString(file, toLine(entry) + System.lineSeparator(), StandardOpenOption.CREATE,
StandardOpenOption.APPEND);
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, "audit-log", e);
}
}
public List<AuditLogEntry> list()
{
Path file = logFile();
if (!Files.exists(file))
{
return List.of();
}
try
{
return Files.readAllLines(file).stream()
.filter(line -> !line.isBlank())
.map(this::fromLine)
.toList();
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, "audit-log", e);
}
}
private Path logFile()
{
return Path.of(workdir, ".audit", "audit.log");
}
private String toLine(AuditLogEntry entry)
{
return String.join("\t",
entry.timestamp().toString(),
entry.action().name(),
entry.resourceType().name(),
escape(entry.bucket()),
nullableField(entry.key()),
nullableField(entry.principal()));
}
private AuditLogEntry fromLine(String line)
{
String[] fields = line.split("\t", 6);
return new AuditLogEntry(
Instant.parse(fields[0]),
AuditAction.valueOf(fields[1]),
AuditResourceType.valueOf(fields[2]),
unescape(fields[3]),
parseNullableField(fields[4]),
parseNullableField(fields[5]));
}
private String nullableField(String value)
{
if (value == null)
{
return "";
}
return escape(value);
}
private String parseNullableField(String field)
{
if (field.isEmpty())
{
return null;
}
return unescape(field);
}
private String escape(String value)
{
return value
.replace("\\", "\\\\")
.replace("\t", "\\t")
.replace("\n", "\\n")
.replace("\r", "\\r");
}
private String unescape(String value)
{
StringBuilder result = new StringBuilder();
for (int i = 0; i < value.length(); i++)
{
char c = value.charAt(i);
if (c == '\\' && i + 1 < value.length())
{
i++;
result.append(unescapedChar(value.charAt(i)));
}
else
{
result.append(c);
}
}
return result.toString();
}
private char unescapedChar(char c)
{
if (c == 't')
{
return '\t';
}
if (c == 'n')
{
return '\n';
}
if (c == 'r')
{
return '\r';
}
return c;
}
}
@@ -0,0 +1,27 @@
package dev.dinauer.r9t.core.audit;
import io.quarkus.security.Authenticated;
import jakarta.inject.Inject;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
@Path("/_audit-logs")
@Authenticated
public class AuditLogResource
{
@Inject
AuditLogService auditLogService;
@Inject
AuditLogJsonMapper auditLogJsonMapper;
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response list()
{
return Response.ok(auditLogJsonMapper.map(auditLogService.list())).build();
}
}
@@ -0,0 +1,41 @@
package dev.dinauer.r9t.core.audit;
import io.quarkus.security.identity.SecurityIdentity;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import java.time.Instant;
import java.util.Comparator;
import java.util.List;
@ApplicationScoped
public class AuditLogService
{
@Inject
AuditLogRepo auditLogRepo;
@Inject
SecurityIdentity securityIdentity;
public void record(AuditAction action, AuditResourceType resourceType, String bucket, String key)
{
AuditLogEntry entry = new AuditLogEntry(Instant.now(), action, resourceType, bucket, key, principal());
auditLogRepo.append(entry);
}
public List<AuditLogEntry> list()
{
return auditLogRepo.list().stream()
.sorted(Comparator.comparing(AuditLogEntry::timestamp).reversed())
.toList();
}
private String principal()
{
if (securityIdentity.getPrincipal() != null)
{
return securityIdentity.getPrincipal().getName();
}
return null;
}
}
@@ -0,0 +1,7 @@
package dev.dinauer.r9t.core.audit;
public enum AuditResourceType
{
BUCKET,
OBJECT
}
@@ -1,17 +1,23 @@
package dev.dinauer.r9t.core.bucket; package dev.dinauer.r9t.core.bucket;
import dev.dinauer.r9t.core.s3.FileAttributes;
import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception; import dev.dinauer.r9t.core.s3.S3Exception;
import dev.dinauer.r9t.core.s3.Tag;
import dev.dinauer.r9t.core.s3.TagCodec;
import jakarta.annotation.PostConstruct; import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import org.eclipse.microprofile.config.inject.ConfigProperty; import org.eclipse.microprofile.config.inject.ConfigProperty;
import java.io.IOException; import java.io.IOException;
import java.nio.file.DirectoryNotEmptyException; import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
import java.util.List; import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Stream; import java.util.stream.Stream;
@ApplicationScoped @ApplicationScoped
@@ -97,6 +103,77 @@ public class BucketRepo
} }
} }
public List<Tag> getTags(String bucket)
{
Path target = resolve(bucket);
try
{
if (FileAttributes.exists(target, FileAttributes.TAGS))
{
return TagCodec.parseQueryString(FileAttributes.read(target, FileAttributes.TAGS));
}
return List.of();
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e);
}
}
public void putTags(String bucket, List<Tag> tags)
{
Path target = resolve(bucket);
try
{
FileAttributes.write(target, FileAttributes.TAGS, TagCodec.toQueryString(tags));
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e);
}
}
public void deleteTags(String bucket)
{
Path target = resolve(bucket);
try
{
if (FileAttributes.exists(target, FileAttributes.TAGS))
{
FileAttributes.delete(target, FileAttributes.TAGS);
}
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e);
}
}
public BucketStats stats(String bucket)
{
Path bucketPath = resolve(bucket);
AtomicLong objectCount = new AtomicLong();
AtomicLong totalSize = new AtomicLong();
try
{
Files.walkFileTree(bucketPath, new SimpleFileVisitor<>()
{
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attributes)
{
objectCount.incrementAndGet();
totalSize.addAndGet(attributes.size());
return FileVisitResult.CONTINUE;
}
});
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e);
}
return new BucketStats(bucket, objectCount.get(), totalSize.get());
}
private Bucket toBucket(Path dir) private Bucket toBucket(Path dir)
{ {
try try
@@ -1,15 +1,22 @@
package dev.dinauer.r9t.core.bucket; package dev.dinauer.r9t.core.bucket;
import dev.dinauer.r9t.core.s3.Tag;
import io.quarkus.security.Authenticated; import io.quarkus.security.Authenticated;
import jakarta.inject.Inject; import jakarta.inject.Inject;
import jakarta.ws.rs.Consumes;
import jakarta.ws.rs.DELETE; import jakarta.ws.rs.DELETE;
import jakarta.ws.rs.GET; import jakarta.ws.rs.GET;
import jakarta.ws.rs.PUT; import jakarta.ws.rs.PUT;
import jakarta.ws.rs.Path; import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam; import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces; import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.Context;
import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.UriInfo;
import java.io.InputStream;
import java.util.List;
@Path("/") @Path("/")
@Authenticated @Authenticated
@@ -30,16 +37,28 @@ public class BucketResource
@PUT @PUT
@Path("{bucket}") @Path("{bucket}")
public Response create(@PathParam("bucket") String bucket) @Consumes(MediaType.WILDCARD)
public Response create(@PathParam("bucket") String bucket, @Context UriInfo uriInfo, InputStream body)
{ {
if (uriInfo.getQueryParameters().containsKey("tagging"))
{
List<Tag> tags = bucketXmlMapper.mapTagging(body);
bucketService.putTags(bucket, tags);
return Response.ok().build();
}
bucketService.create(bucket); bucketService.create(bucket);
return Response.ok().build(); return Response.ok().build();
} }
@DELETE @DELETE
@Path("{bucket}") @Path("{bucket}")
public Response delete(@PathParam("bucket") String bucket) public Response delete(@PathParam("bucket") String bucket, @Context UriInfo uriInfo)
{ {
if (uriInfo.getQueryParameters().containsKey("tagging"))
{
bucketService.deleteTags(bucket);
return Response.noContent().build();
}
bucketService.delete(bucket); bucketService.delete(bucket);
return Response.noContent().build(); return Response.noContent().build();
} }
@@ -1,7 +1,12 @@
package dev.dinauer.r9t.core.bucket; package dev.dinauer.r9t.core.bucket;
import dev.dinauer.r9t.core.audit.AuditAction;
import dev.dinauer.r9t.core.audit.AuditLogService;
import dev.dinauer.r9t.core.audit.AuditResourceType;
import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception; import dev.dinauer.r9t.core.s3.S3Exception;
import dev.dinauer.r9t.core.s3.Tag;
import dev.dinauer.r9t.core.s3.TagValidator;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject; import jakarta.inject.Inject;
@@ -16,11 +21,19 @@ public class BucketService
@Inject @Inject
BucketRepo bucketRepo; BucketRepo bucketRepo;
@Inject
AuditLogService auditLogService;
public List<Bucket> list() public List<Bucket> list()
{ {
return bucketRepo.list(); return bucketRepo.list();
} }
public List<BucketStats> stats()
{
return bucketRepo.list().stream().map(bucket -> bucketRepo.stats(bucket.name())).toList();
}
public boolean exists(String name) public boolean exists(String name)
{ {
return bucketRepo.exists(name); return bucketRepo.exists(name);
@@ -34,12 +47,14 @@ public class BucketService
throw new S3Exception(S3ErrorCode.BUCKET_ALREADY_EXISTS, name); throw new S3Exception(S3ErrorCode.BUCKET_ALREADY_EXISTS, name);
} }
bucketRepo.create(name); bucketRepo.create(name);
auditLogService.record(AuditAction.CREATE, AuditResourceType.BUCKET, name, null);
} }
public void delete(String name) public void delete(String name)
{ {
require(name); require(name);
bucketRepo.delete(name); bucketRepo.delete(name);
auditLogService.record(AuditAction.DELETE, AuditResourceType.BUCKET, name, null);
} }
public void require(String name) public void require(String name)
@@ -50,6 +65,25 @@ public class BucketService
} }
} }
public List<Tag> getTags(String name)
{
require(name);
return bucketRepo.getTags(name);
}
public void putTags(String name, List<Tag> tags)
{
require(name);
TagValidator.validate(tags);
bucketRepo.putTags(name, tags);
}
public void deleteTags(String name)
{
require(name);
bucketRepo.deleteTags(name);
}
private void validate(String name) private void validate(String name)
{ {
if (!NAME_PATTERN.matcher(name).matches()) if (!NAME_PATTERN.matcher(name).matches())
@@ -0,0 +1,5 @@
package dev.dinauer.r9t.core.bucket;
public record BucketStats(String name, long objectCount, long totalSize)
{
}
@@ -0,0 +1,33 @@
package dev.dinauer.r9t.core.bucket;
import jakarta.enterprise.context.ApplicationScoped;
import java.util.List;
import java.util.stream.Collectors;
@ApplicationScoped
public class BucketStatsJsonMapper
{
public String map(List<BucketStats> input)
{
return "[" + input.stream().map(this::map).collect(Collectors.joining(",")) + "]";
}
private String map(BucketStats input)
{
return "{"
+ "\"name\":\"" + escape(input.name()) + "\","
+ "\"objectCount\":" + input.objectCount() + ","
+ "\"totalSize\":" + input.totalSize()
+ "}";
}
private String escape(String value)
{
return value
.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r");
}
}
@@ -0,0 +1,27 @@
package dev.dinauer.r9t.core.bucket;
import io.quarkus.security.Authenticated;
import jakarta.inject.Inject;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
@Path("/_bucket-stats")
@Authenticated
public class BucketStatsResource
{
@Inject
BucketService bucketService;
@Inject
BucketStatsJsonMapper bucketStatsJsonMapper;
@GET
@Produces(MediaType.APPLICATION_JSON)
public Response list()
{
return Response.ok(bucketStatsJsonMapper.map(bucketService.stats())).build();
}
}
@@ -1,8 +1,11 @@
package dev.dinauer.r9t.core.bucket; package dev.dinauer.r9t.core.bucket;
import dev.dinauer.r9t.core.s3.S3Xml; import dev.dinauer.r9t.core.s3.S3Xml;
import dev.dinauer.r9t.core.s3.Tag;
import dev.dinauer.r9t.core.s3.TagXml;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import java.io.InputStream;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -27,4 +30,14 @@ public class BucketXmlMapper
+ " </Buckets>\n" + " </Buckets>\n"
+ "</ListAllMyBucketsResult>"; + "</ListAllMyBucketsResult>";
} }
public String mapTagging(List<Tag> tags)
{
return TagXml.map(tags);
}
public List<Tag> mapTagging(InputStream body)
{
return TagXml.parse(body);
}
} }
@@ -1,7 +1,10 @@
package dev.dinauer.r9t.core.multipart; package dev.dinauer.r9t.core.multipart;
import java.time.Instant; import dev.dinauer.r9t.core.s3.Tag;
public record MultipartUpload(String uploadId, String bucket, String key, String contentType, Instant initiated) import java.time.Instant;
import java.util.List;
public record MultipartUpload(String uploadId, String bucket, String key, String contentType, Instant initiated, List<Tag> tags)
{ {
} }
@@ -5,6 +5,8 @@ import dev.dinauer.r9t.core.s3.DigestCopy;
import dev.dinauer.r9t.core.s3.FileAttributes; import dev.dinauer.r9t.core.s3.FileAttributes;
import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception; import dev.dinauer.r9t.core.s3.S3Exception;
import dev.dinauer.r9t.core.s3.Tag;
import dev.dinauer.r9t.core.s3.TagCodec;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject; import jakarta.inject.Inject;
@@ -14,6 +16,7 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.Instant; import java.time.Instant;
import java.util.Comparator; import java.util.Comparator;
import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.stream.Stream; import java.util.stream.Stream;
@@ -43,7 +46,7 @@ public class MultipartUploadRepo
return Files.isDirectory(resolve(uploadId)); return Files.isDirectory(resolve(uploadId));
} }
public MultipartUpload create(String bucket, String key, String contentType) public MultipartUpload create(String bucket, String key, String contentType, List<Tag> tags)
{ {
String uploadId = UUID.randomUUID().toString().replace("-", ""); String uploadId = UUID.randomUUID().toString().replace("-", "");
Path dir = resolve(uploadId); Path dir = resolve(uploadId);
@@ -55,12 +58,13 @@ public class MultipartUploadRepo
FileAttributes.write(dir, KEY_ATTRIBUTE, key); FileAttributes.write(dir, KEY_ATTRIBUTE, key);
FileAttributes.write(dir, FileAttributes.CONTENT_TYPE, contentType); FileAttributes.write(dir, FileAttributes.CONTENT_TYPE, contentType);
FileAttributes.write(dir, INITIATED_ATTRIBUTE, initiated.toString()); FileAttributes.write(dir, INITIATED_ATTRIBUTE, initiated.toString());
FileAttributes.write(dir, FileAttributes.TAGS, TagCodec.toQueryString(tags));
} }
catch (IOException e) catch (IOException e)
{ {
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
} }
return new MultipartUpload(uploadId, bucket, key, contentType, initiated); return new MultipartUpload(uploadId, bucket, key, contentType, initiated, tags);
} }
public MultipartUpload metadata(String uploadId) public MultipartUpload metadata(String uploadId)
@@ -72,7 +76,8 @@ public class MultipartUploadRepo
String key = FileAttributes.read(dir, KEY_ATTRIBUTE); String key = FileAttributes.read(dir, KEY_ATTRIBUTE);
String contentType = FileAttributes.read(dir, FileAttributes.CONTENT_TYPE); String contentType = FileAttributes.read(dir, FileAttributes.CONTENT_TYPE);
Instant initiated = Instant.parse(FileAttributes.read(dir, INITIATED_ATTRIBUTE)); Instant initiated = Instant.parse(FileAttributes.read(dir, INITIATED_ATTRIBUTE));
return new MultipartUpload(uploadId, bucket, key, contentType, initiated); List<Tag> tags = TagCodec.parseQueryString(FileAttributes.read(dir, FileAttributes.TAGS));
return new MultipartUpload(uploadId, bucket, key, contentType, initiated, tags);
} }
catch (IOException e) catch (IOException e)
{ {
@@ -1,11 +1,16 @@
package dev.dinauer.r9t.core.multipart; package dev.dinauer.r9t.core.multipart;
import dev.dinauer.r9t.core.audit.AuditAction;
import dev.dinauer.r9t.core.audit.AuditLogService;
import dev.dinauer.r9t.core.audit.AuditResourceType;
import dev.dinauer.r9t.core.bucket.BucketService; import dev.dinauer.r9t.core.bucket.BucketService;
import dev.dinauer.r9t.core.object.ObjectRepo; import dev.dinauer.r9t.core.object.ObjectRepo;
import dev.dinauer.r9t.core.object.S3Object; import dev.dinauer.r9t.core.object.S3Object;
import dev.dinauer.r9t.core.s3.DigestCopy; import dev.dinauer.r9t.core.s3.DigestCopy;
import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception; import dev.dinauer.r9t.core.s3.S3Exception;
import dev.dinauer.r9t.core.s3.Tag;
import dev.dinauer.r9t.core.s3.TagValidator;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject; import jakarta.inject.Inject;
@@ -34,7 +39,10 @@ public class MultipartUploadService
@Inject @Inject
BucketService bucketService; BucketService bucketService;
public MultipartUpload initiate(String bucket, String key, String contentType) @Inject
AuditLogService auditLogService;
public MultipartUpload initiate(String bucket, String key, String contentType, List<Tag> tags)
{ {
bucketService.require(bucket); bucketService.require(bucket);
String resolvedContentType = contentType; String resolvedContentType = contentType;
@@ -42,7 +50,8 @@ public class MultipartUploadService
{ {
resolvedContentType = "application/octet-stream"; resolvedContentType = "application/octet-stream";
} }
return multipartUploadRepo.create(bucket, key, resolvedContentType); TagValidator.validate(tags);
return multipartUploadRepo.create(bucket, key, resolvedContentType, tags);
} }
public String uploadPart(String bucket, String key, String uploadId, Integer partNumber, InputStream data) public String uploadPart(String bucket, String key, String uploadId, Integer partNumber, InputStream data)
@@ -72,7 +81,12 @@ public class MultipartUploadService
{ {
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e);
} }
if (!upload.tags().isEmpty())
{
objectRepo.putTags(bucket, key, upload.tags());
}
multipartUploadRepo.delete(uploadId); multipartUploadRepo.delete(uploadId);
auditLogService.record(AuditAction.CREATE, AuditResourceType.OBJECT, bucket, key);
return objectRepo.metadata(bucket, key); return objectRepo.metadata(bucket, key);
} }
@@ -0,0 +1,7 @@
package dev.dinauer.r9t.core.object;
import java.util.List;
public record ListObjectsResult(List<S3Object> contents, List<String> commonPrefixes)
{
}
@@ -5,6 +5,8 @@ import dev.dinauer.r9t.core.s3.DigestCopy;
import dev.dinauer.r9t.core.s3.FileAttributes; import dev.dinauer.r9t.core.s3.FileAttributes;
import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception; import dev.dinauer.r9t.core.s3.S3Exception;
import dev.dinauer.r9t.core.s3.Tag;
import dev.dinauer.r9t.core.s3.TagCodec;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject; import jakarta.inject.Inject;
@@ -14,7 +16,10 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.stream.Stream; import java.util.stream.Stream;
@ApplicationScoped @ApplicationScoped
@@ -48,6 +53,7 @@ public class ObjectRepo
String etag = DigestCopy.copyAndHashMd5(data, target); String etag = DigestCopy.copyAndHashMd5(data, target);
FileAttributes.write(target, FileAttributes.ETAG, etag); FileAttributes.write(target, FileAttributes.ETAG, etag);
FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType); FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType);
clearTags(target);
return etag; return etag;
} }
catch (IOException e) catch (IOException e)
@@ -65,6 +71,7 @@ public class ObjectRepo
Files.copy(data, target, StandardCopyOption.REPLACE_EXISTING); Files.copy(data, target, StandardCopyOption.REPLACE_EXISTING);
FileAttributes.write(target, FileAttributes.ETAG, etag); FileAttributes.write(target, FileAttributes.ETAG, etag);
FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType); FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType);
clearTags(target);
} }
catch (IOException e) catch (IOException e)
{ {
@@ -72,6 +79,57 @@ public class ObjectRepo
} }
} }
public List<Tag> getTags(String bucket, String key)
{
Path target = resolve(bucket, key);
try
{
if (FileAttributes.exists(target, FileAttributes.TAGS))
{
return TagCodec.parseQueryString(FileAttributes.read(target, FileAttributes.TAGS));
}
return List.of();
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e);
}
}
public void putTags(String bucket, String key, List<Tag> tags)
{
Path target = resolve(bucket, key);
try
{
FileAttributes.write(target, FileAttributes.TAGS, TagCodec.toQueryString(tags));
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e);
}
}
public void deleteTags(String bucket, String key)
{
Path target = resolve(bucket, key);
try
{
clearTags(target);
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e);
}
}
private void clearTags(Path target) throws IOException
{
if (FileAttributes.exists(target, FileAttributes.TAGS))
{
FileAttributes.delete(target, FileAttributes.TAGS);
}
}
public S3Object metadata(String bucket, String key) public S3Object metadata(String bucket, String key)
{ {
Path target = resolve(bucket, key); Path target = resolve(bucket, key);
@@ -105,14 +163,17 @@ public class ObjectRepo
} }
} }
public List<S3Object> list(String bucket) public List<S3Object> searchByTags(String bucket, Map<String, String> tagFilters)
{ {
Path bucketPath = bucketRepo.resolve(bucket); Path bucketPath = bucketRepo.resolve(bucket);
try (Stream<Path> entries = Files.walk(bucketPath)) try (Stream<Path> entries = Files.walk(bucketPath))
{ {
return entries return entries
.filter(Files::isRegularFile) .filter(Files::isRegularFile)
.map(path -> metadata(bucket, bucketPath.relativize(path).toString())) .map(path -> bucketPath.relativize(path).toString())
.filter(key -> matchesAllTags(bucket, key, tagFilters))
.sorted()
.map(key -> metadata(bucket, key))
.toList(); .toList();
} }
catch (IOException e) catch (IOException e)
@@ -120,4 +181,84 @@ public class ObjectRepo
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e);
} }
} }
private boolean matchesAllTags(String bucket, String key, Map<String, String> tagFilters)
{
List<Tag> tags = getTags(bucket, key);
for (Map.Entry<String, String> filter : tagFilters.entrySet())
{
if (!matchesTag(tags, filter.getKey(), filter.getValue()))
{
return false;
}
}
return true;
}
private boolean matchesTag(List<Tag> tags, String key, String value)
{
return tags.stream().anyMatch(tag -> tag.key().equals(key) && tag.value().equals(value));
}
public ListObjectsResult list(String bucket, String prefix, String delimiter)
{
Path bucketPath = bucketRepo.resolve(bucket);
try (Stream<Path> entries = Files.walk(bucketPath))
{
List<String> keys = entries
.filter(Files::isRegularFile)
.map(path -> bucketPath.relativize(path).toString())
.filter(key -> matchesPrefix(key, prefix))
.sorted()
.toList();
return group(bucket, keys, normalizedPrefix(prefix), delimiter);
}
catch (IOException e)
{
throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e);
}
}
private boolean matchesPrefix(String key, String prefix)
{
if (prefix == null)
{
return true;
}
return key.startsWith(prefix);
}
private String normalizedPrefix(String prefix)
{
if (prefix == null)
{
return "";
}
return prefix;
}
private ListObjectsResult group(String bucket, List<String> keys, String prefix, String delimiter)
{
if (delimiter == null)
{
List<S3Object> contents = keys.stream().map(key -> metadata(bucket, key)).toList();
return new ListObjectsResult(contents, List.of());
}
List<S3Object> contents = new ArrayList<>();
LinkedHashSet<String> commonPrefixes = new LinkedHashSet<>();
for (String key : keys)
{
String rest = key.substring(prefix.length());
int delimiterIndex = rest.indexOf(delimiter);
if (delimiterIndex == -1)
{
contents.add(metadata(bucket, key));
}
else
{
commonPrefixes.add(prefix + rest.substring(0, delimiterIndex + delimiter.length()));
}
}
return new ListObjectsResult(contents, List.copyOf(commonPrefixes));
}
} }
@@ -1,10 +1,14 @@
package dev.dinauer.r9t.core.object; package dev.dinauer.r9t.core.object;
import dev.dinauer.r9t.core.auth.AwsChunkedInputStream; import dev.dinauer.r9t.core.auth.AwsChunkedInputStream;
import dev.dinauer.r9t.core.bucket.BucketService;
import dev.dinauer.r9t.core.bucket.BucketXmlMapper;
import dev.dinauer.r9t.core.multipart.CompletedPart; import dev.dinauer.r9t.core.multipart.CompletedPart;
import dev.dinauer.r9t.core.multipart.MultipartUpload; import dev.dinauer.r9t.core.multipart.MultipartUpload;
import dev.dinauer.r9t.core.multipart.MultipartUploadService; import dev.dinauer.r9t.core.multipart.MultipartUploadService;
import dev.dinauer.r9t.core.multipart.MultipartUploadXmlMapper; import dev.dinauer.r9t.core.multipart.MultipartUploadXmlMapper;
import dev.dinauer.r9t.core.s3.Tag;
import dev.dinauer.r9t.core.s3.TagCodec;
import io.quarkus.security.Authenticated; import io.quarkus.security.Authenticated;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import jakarta.inject.Inject; import jakarta.inject.Inject;
@@ -26,12 +30,16 @@ import jakarta.ws.rs.core.UriInfo;
import java.io.InputStream; import java.io.InputStream;
import java.util.Date; import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List; import java.util.List;
import java.util.Map;
@Path("/{bucket}") @Path("/{bucket}")
@Authenticated @Authenticated
public class ObjectResource public class ObjectResource
{ {
private static final String TAG_QUERY_PREFIX = "x-tag-";
@Inject @Inject
ObjectService objectService; ObjectService objectService;
@@ -44,11 +52,45 @@ public class ObjectResource
@Inject @Inject
MultipartUploadXmlMapper multipartUploadXmlMapper; MultipartUploadXmlMapper multipartUploadXmlMapper;
@Inject
BucketService bucketService;
@Inject
BucketXmlMapper bucketXmlMapper;
@GET @GET
@Produces(MediaType.APPLICATION_XML) @Produces(MediaType.APPLICATION_XML)
public Response list(@PathParam("bucket") String bucket) public Response list(@PathParam("bucket") String bucket, @QueryParam("prefix") String prefix,
@QueryParam("delimiter") String delimiter, @Context UriInfo uriInfo)
{ {
return Response.ok(objectXmlMapper.map(bucket, objectService.list(bucket))).build(); if (uriInfo.getQueryParameters().containsKey("tagging"))
{
List<Tag> tags = bucketService.getTags(bucket);
return Response.ok(bucketXmlMapper.mapTagging(tags)).build();
}
Map<String, String> tagFilters = tagFilters(uriInfo);
if (!tagFilters.isEmpty())
{
List<S3Object> matches = objectService.searchByTags(bucket, tagFilters);
ListObjectsResult result = new ListObjectsResult(matches, List.of());
return Response.ok(objectXmlMapper.map(bucket, null, null, result)).build();
}
ListObjectsResult result = objectService.list(bucket, prefix, delimiter);
return Response.ok(objectXmlMapper.map(bucket, prefix, delimiter, result)).build();
}
private Map<String, String> tagFilters(UriInfo uriInfo)
{
Map<String, String> filters = new LinkedHashMap<>();
for (Map.Entry<String, List<String>> param : uriInfo.getQueryParameters().entrySet())
{
if (param.getKey().startsWith(TAG_QUERY_PREFIX))
{
String tagName = param.getKey().substring(TAG_QUERY_PREFIX.length());
filters.put(tagName, param.getValue().get(0));
}
}
return filters;
} }
@PUT @PUT
@@ -56,9 +98,17 @@ public class ObjectResource
@Consumes(MediaType.WILDCARD) @Consumes(MediaType.WILDCARD)
public Response put(@PathParam("bucket") String bucket, @PathParam("key") String key, public Response put(@PathParam("bucket") String bucket, @PathParam("key") String key,
@QueryParam("uploadId") String uploadId, @QueryParam("partNumber") Integer partNumber, @QueryParam("uploadId") String uploadId, @QueryParam("partNumber") Integer partNumber,
@Context UriInfo uriInfo,
@HeaderParam("Content-Type") String contentType, @HeaderParam("Content-Type") String contentType,
@HeaderParam("X-Amz-Content-Sha256") String contentSha256, InputStream body) @HeaderParam("X-Amz-Content-Sha256") String contentSha256,
@HeaderParam("x-amz-tagging") String tagging, InputStream body)
{ {
if (uriInfo.getQueryParameters().containsKey("tagging"))
{
List<Tag> tags = objectXmlMapper.mapTagging(body);
objectService.putTags(bucket, key, tags);
return Response.ok().build();
}
InputStream payload = body; InputStream payload = body;
if (StringUtils.startsWith(contentSha256, "STREAMING-")) if (StringUtils.startsWith(contentSha256, "STREAMING-"))
{ {
@@ -70,6 +120,10 @@ public class ObjectResource
return Response.ok().header("ETag", "\"" + partEtag + "\"").build(); return Response.ok().header("ETag", "\"" + partEtag + "\"").build();
} }
String etag = objectService.put(bucket, key, contentType, payload); String etag = objectService.put(bucket, key, contentType, payload);
if (tagging != null)
{
objectService.putTags(bucket, key, TagCodec.parseQueryString(tagging));
}
return Response.ok().header("ETag", "\"" + etag + "\"").build(); return Response.ok().header("ETag", "\"" + etag + "\"").build();
} }
@@ -79,11 +133,13 @@ public class ObjectResource
@Produces(MediaType.APPLICATION_XML) @Produces(MediaType.APPLICATION_XML)
public Response post(@PathParam("bucket") String bucket, @PathParam("key") String key, public Response post(@PathParam("bucket") String bucket, @PathParam("key") String key,
@Context UriInfo uriInfo, @QueryParam("uploadId") String uploadId, @Context UriInfo uriInfo, @QueryParam("uploadId") String uploadId,
@HeaderParam("Content-Type") String contentType, InputStream body) @HeaderParam("Content-Type") String contentType,
@HeaderParam("x-amz-tagging") String tagging, InputStream body)
{ {
if (uriInfo.getQueryParameters().containsKey("uploads")) if (uriInfo.getQueryParameters().containsKey("uploads"))
{ {
MultipartUpload upload = multipartUploadService.initiate(bucket, key, contentType); List<Tag> tags = TagCodec.parseQueryString(tagging);
MultipartUpload upload = multipartUploadService.initiate(bucket, key, contentType, tags);
return Response.ok(multipartUploadXmlMapper.map(upload)).build(); return Response.ok(multipartUploadXmlMapper.map(upload)).build();
} }
List<CompletedPart> parts = multipartUploadXmlMapper.map(body); List<CompletedPart> parts = multipartUploadXmlMapper.map(body);
@@ -93,8 +149,13 @@ public class ObjectResource
@GET @GET
@Path("{key:.+}") @Path("{key:.+}")
public Response get(@PathParam("bucket") String bucket, @PathParam("key") String key) public Response get(@PathParam("bucket") String bucket, @PathParam("key") String key, @Context UriInfo uriInfo)
{ {
if (uriInfo.getQueryParameters().containsKey("tagging"))
{
List<Tag> tags = objectService.getTags(bucket, key);
return Response.ok(objectXmlMapper.mapTagging(tags)).type(MediaType.APPLICATION_XML).build();
}
S3Object metadata = objectService.metadata(bucket, key); S3Object metadata = objectService.metadata(bucket, key);
return Response.ok(objectService.file(bucket, key).toFile()) return Response.ok(objectService.file(bucket, key).toFile())
.type(metadata.contentType()) .type(metadata.contentType())
@@ -119,8 +180,13 @@ public class ObjectResource
@DELETE @DELETE
@Path("{key:.+}") @Path("{key:.+}")
public Response delete(@PathParam("bucket") String bucket, @PathParam("key") String key, public Response delete(@PathParam("bucket") String bucket, @PathParam("key") String key,
@QueryParam("uploadId") String uploadId) @QueryParam("uploadId") String uploadId, @Context UriInfo uriInfo)
{ {
if (uriInfo.getQueryParameters().containsKey("tagging"))
{
objectService.deleteTags(bucket, key);
return Response.noContent().build();
}
if (uploadId != null) if (uploadId != null)
{ {
multipartUploadService.abort(bucket, key, uploadId); multipartUploadService.abort(bucket, key, uploadId);
@@ -1,14 +1,20 @@
package dev.dinauer.r9t.core.object; package dev.dinauer.r9t.core.object;
import dev.dinauer.r9t.core.audit.AuditAction;
import dev.dinauer.r9t.core.audit.AuditLogService;
import dev.dinauer.r9t.core.audit.AuditResourceType;
import dev.dinauer.r9t.core.bucket.BucketService; import dev.dinauer.r9t.core.bucket.BucketService;
import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3ErrorCode;
import dev.dinauer.r9t.core.s3.S3Exception; import dev.dinauer.r9t.core.s3.S3Exception;
import dev.dinauer.r9t.core.s3.Tag;
import dev.dinauer.r9t.core.s3.TagValidator;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject; import jakarta.inject.Inject;
import java.io.InputStream; import java.io.InputStream;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.List; import java.util.List;
import java.util.Map;
@ApplicationScoped @ApplicationScoped
public class ObjectService public class ObjectService
@@ -19,10 +25,23 @@ public class ObjectService
@Inject @Inject
BucketService bucketService; BucketService bucketService;
public List<S3Object> list(String bucket) @Inject
AuditLogService auditLogService;
public ListObjectsResult list(String bucket, String prefix, String delimiter)
{ {
bucketService.require(bucket); bucketService.require(bucket);
return objectRepo.list(bucket); return objectRepo.list(bucket, prefix, delimiter);
}
public List<S3Object> searchByTags(String bucket, Map<String, String> tagFilters)
{
bucketService.require(bucket);
if (tagFilters.isEmpty())
{
throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, "x-tag-*");
}
return objectRepo.searchByTags(bucket, tagFilters);
} }
public String put(String bucket, String key, String contentType, InputStream data) public String put(String bucket, String key, String contentType, InputStream data)
@@ -33,7 +52,9 @@ public class ObjectService
{ {
resolvedContentType = "application/octet-stream"; resolvedContentType = "application/octet-stream";
} }
return objectRepo.put(bucket, key, resolvedContentType, data); String etag = objectRepo.put(bucket, key, resolvedContentType, data);
auditLogService.record(AuditAction.CREATE, AuditResourceType.OBJECT, bucket, key);
return etag;
} }
public S3Object metadata(String bucket, String key) public S3Object metadata(String bucket, String key)
@@ -56,5 +77,25 @@ public class ObjectService
{ {
bucketService.require(bucket); bucketService.require(bucket);
objectRepo.delete(bucket, key); objectRepo.delete(bucket, key);
auditLogService.record(AuditAction.DELETE, AuditResourceType.OBJECT, bucket, key);
}
public List<Tag> getTags(String bucket, String key)
{
metadata(bucket, key);
return objectRepo.getTags(bucket, key);
}
public void putTags(String bucket, String key, List<Tag> tags)
{
metadata(bucket, key);
TagValidator.validate(tags);
objectRepo.putTags(bucket, key, tags);
}
public void deleteTags(String bucket, String key)
{
metadata(bucket, key);
objectRepo.deleteTags(bucket, key);
} }
} }
@@ -1,8 +1,11 @@
package dev.dinauer.r9t.core.object; package dev.dinauer.r9t.core.object;
import dev.dinauer.r9t.core.s3.S3Xml; import dev.dinauer.r9t.core.s3.S3Xml;
import dev.dinauer.r9t.core.s3.Tag;
import dev.dinauer.r9t.core.s3.TagXml;
import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.context.ApplicationScoped;
import java.io.InputStream;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@@ -19,14 +22,46 @@ public class ObjectXmlMapper
+ "</Contents>"; + "</Contents>";
} }
public String map(String bucket, List<S3Object> input) public String map(String bucket, String prefix, String delimiter, ListObjectsResult input)
{ {
String contents = input.stream().map(this::map).collect(Collectors.joining()); String contents = input.contents().stream().map(this::map).collect(Collectors.joining());
String commonPrefixes = input.commonPrefixes().stream().map(this::mapCommonPrefix).collect(Collectors.joining());
String prefixTag = "";
if (prefix != null)
{
prefixTag = " <Prefix>" + S3Xml.escape(prefix) + "</Prefix>\n";
}
String delimiterTag = "";
if (delimiter != null)
{
delimiterTag = " <Delimiter>" + S3Xml.escape(delimiter) + "</Delimiter>\n";
}
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n" + "<ListBucketResult xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n"
+ " <Name>" + S3Xml.escape(bucket) + "</Name>\n" + " <Name>" + S3Xml.escape(bucket) + "</Name>\n"
+ prefixTag
+ delimiterTag
+ " <KeyCount>" + input.contents().size() + "</KeyCount>\n"
+ " <IsTruncated>false</IsTruncated>\n" + " <IsTruncated>false</IsTruncated>\n"
+ contents + contents
+ commonPrefixes
+ "</ListBucketResult>"; + "</ListBucketResult>";
} }
public String mapTagging(List<Tag> tags)
{
return TagXml.map(tags);
}
public List<Tag> mapTagging(InputStream body)
{
return TagXml.parse(body);
}
private String mapCommonPrefix(String prefix)
{
return "<CommonPrefixes>\n"
+ " <Prefix>" + S3Xml.escape(prefix) + "</Prefix>\n"
+ "</CommonPrefixes>";
}
} }
@@ -11,6 +11,7 @@ public final class FileAttributes
{ {
public static final String ETAG = "user.etag"; public static final String ETAG = "user.etag";
public static final String CONTENT_TYPE = "user.contentType"; public static final String CONTENT_TYPE = "user.contentType";
public static final String TAGS = "user.tags";
private FileAttributes() private FileAttributes()
{ {
@@ -30,4 +31,16 @@ public final class FileAttributes
buffer.flip(); buffer.flip();
return StandardCharsets.UTF_8.decode(buffer).toString(); return StandardCharsets.UTF_8.decode(buffer).toString();
} }
public static boolean exists(Path file, String name) throws IOException
{
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
return view.list().contains(name);
}
public static void delete(Path file, String name) throws IOException
{
UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class);
view.delete(name);
}
} }
@@ -11,6 +11,7 @@ public enum S3ErrorCode
NO_SUCH_UPLOAD("NoSuchUpload", 404, "The specified multipart upload does not exist"), NO_SUCH_UPLOAD("NoSuchUpload", 404, "The specified multipart upload does not exist"),
INVALID_PART("InvalidPart", 400, "One or more of the specified parts could not be found"), INVALID_PART("InvalidPart", 400, "One or more of the specified parts could not be found"),
MALFORMED_XML("MalformedXML", 400, "The XML you provided was not well-formed or did not validate against our published schema"), MALFORMED_XML("MalformedXML", 400, "The XML you provided was not well-formed or did not validate against our published schema"),
INVALID_TAG("InvalidTag", 400, "The tag provided was not a valid tag. This error can occur if the tag did not pass input validation."),
INTERNAL_ERROR("InternalError", 500, "We encountered an internal error"); INTERNAL_ERROR("InternalError", 500, "We encountered an internal error");
private final String code; private final String code;
@@ -0,0 +1,5 @@
package dev.dinauer.r9t.core.s3;
public record Tag(String key, String value)
{
}
@@ -0,0 +1,54 @@
package dev.dinauer.r9t.core.s3;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
public final class TagCodec
{
private TagCodec()
{
}
public static List<Tag> parseQueryString(String value)
{
List<Tag> tags = new ArrayList<>();
if (value != null && !value.isBlank())
{
for (String pair : value.split("&"))
{
tags.add(toTag(pair));
}
}
return tags;
}
public static String toQueryString(List<Tag> tags)
{
List<String> pairs = new ArrayList<>();
for (Tag tag : tags)
{
pairs.add(encode(tag.key()) + "=" + encode(tag.value()));
}
return String.join("&", pairs);
}
private static Tag toTag(String pair)
{
String[] parts = pair.split("=", 2);
String key = URLDecoder.decode(parts[0], StandardCharsets.UTF_8);
String value = "";
if (parts.length > 1)
{
value = URLDecoder.decode(parts[1], StandardCharsets.UTF_8);
}
return new Tag(key, value);
}
private static String encode(String value)
{
return URLEncoder.encode(value, StandardCharsets.UTF_8);
}
}
@@ -0,0 +1,38 @@
package dev.dinauer.r9t.core.s3;
import java.util.List;
public final class TagValidator
{
private static final int MAX_TAGS = 10;
private static final int MAX_KEY_LENGTH = 128;
private static final int MAX_VALUE_LENGTH = 256;
private TagValidator()
{
}
public static void validate(List<Tag> tags)
{
if (tags.size() > MAX_TAGS)
{
throw new S3Exception(S3ErrorCode.INVALID_TAG, "TagSet");
}
for (Tag tag : tags)
{
validate(tag);
}
}
private static void validate(Tag tag)
{
if (tag.key() == null || tag.key().isEmpty() || tag.key().length() > MAX_KEY_LENGTH)
{
throw new S3Exception(S3ErrorCode.INVALID_TAG, String.valueOf(tag.key()));
}
if (tag.value() == null || tag.value().length() > MAX_VALUE_LENGTH)
{
throw new S3Exception(S3ErrorCode.INVALID_TAG, tag.key());
}
}
}
@@ -0,0 +1,73 @@
package dev.dinauer.r9t.core.s3;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
public final class TagXml
{
private TagXml()
{
}
public static String map(List<Tag> tags)
{
StringBuilder tagSet = new StringBuilder();
for (Tag tag : tags)
{
tagSet.append(" <Tag>\n")
.append(" <Key>").append(S3Xml.escape(tag.key())).append("</Key>\n")
.append(" <Value>").append(S3Xml.escape(tag.value())).append("</Value>\n")
.append(" </Tag>\n");
}
return "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ "<Tagging xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">\n"
+ " <TagSet>\n"
+ tagSet
+ " </TagSet>\n"
+ "</Tagging>";
}
public static List<Tag> parse(InputStream body)
{
try
{
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true);
DocumentBuilder builder = factory.newDocumentBuilder();
Document document = builder.parse(body);
NodeList tagNodes = document.getElementsByTagName("Tag");
List<Tag> tags = new ArrayList<>();
for (int i = 0; i < tagNodes.getLength(); i++)
{
tags.add(toTag((Element) tagNodes.item(i)));
}
return tags;
}
catch (ParserConfigurationException | SAXException | IOException | NullPointerException e)
{
throw new S3Exception(S3ErrorCode.MALFORMED_XML, "Tagging", e);
}
}
private static Tag toTag(Element tagElement)
{
String key = text(tagElement, "Key");
String value = text(tagElement, "Value");
return new Tag(key, value);
}
private static String text(Element parent, String tagName)
{
return parent.getElementsByTagName(tagName).item(0).getTextContent();
}
}
@@ -59,6 +59,31 @@ public class MultipartUploadTest
assertEquals("hello world", new String(get.body(), StandardCharsets.UTF_8)); assertEquals("hello world", new String(get.body(), StandardCharsets.UTF_8));
} }
@Test
public void initiateWithTaggingAppliesTagsToCompletedObject() throws Exception
{
createBucket();
HttpResponse<String> initiate = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploads="),
new byte[0], "text/plain", "x-amz-tagging", "job=nightly-backup&env=staging");
assertEquals(200, initiate.statusCode());
String uploadId = extract(initiate.body(), "UploadId");
String etag = uploadPart(uploadId, 1, "hello world");
String completeBody = "<CompleteMultipartUpload>"
+ "<Part><PartNumber>1</PartNumber><ETag>\"" + etag + "\"</ETag></Part>"
+ "</CompleteMultipartUpload>";
HttpResponse<String> complete = sendForString("POST", baseUri.resolve(BUCKET + "/" + KEY + "?uploadId=" + uploadId),
completeBody.getBytes(StandardCharsets.UTF_8), null);
assertEquals(200, complete.statusCode());
HttpResponse<String> tagging = sendForString("GET", baseUri.resolve(BUCKET + "/" + KEY + "?tagging="), new byte[0], null);
assertEquals(200, tagging.statusCode());
assertTrue(tagging.body().contains("<Key>job</Key>\n <Value>nightly-backup</Value>"),
"missing tag in tagging response: " + tagging.body());
}
@Test @Test
public void abortDiscardsStagedParts() throws Exception public void abortDiscardsStagedParts() throws Exception
{ {
@@ -132,6 +157,20 @@ public class MultipartUploadTest
return client.send(builder.build(), HttpResponse.BodyHandlers.ofString()); return client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
} }
private HttpResponse<String> sendForString(String method, URI uri, byte[] body, String contentType,
String headerName, String headerValue) throws Exception
{
HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
.method(method, HttpRequest.BodyPublishers.ofByteArray(body))
.header(headerName, headerValue);
if (contentType != null)
{
builder.header("Content-Type", contentType);
}
SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body);
return client.send(builder.build(), HttpResponse.BodyHandlers.ofString());
}
private HttpResponse<byte[]> sendForBytes(String method, URI uri) throws Exception private HttpResponse<byte[]> sendForBytes(String method, URI uri) throws Exception
{ {
HttpRequest.Builder builder = HttpRequest.newBuilder(uri) HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
@@ -0,0 +1,82 @@
package dev.dinauer.r9t.core.object;
import dev.dinauer.r9t.core.auth.SigV4TestSigner;
import io.quarkus.test.common.http.TestHTTPResource;
import io.quarkus.test.junit.QuarkusTest;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@QuarkusTest
public class ObjectTaggingUploadTest
{
private static final String BUCKET = "tagging-upload-test-bucket";
private static final String KEY = "tagging-upload-test.txt";
private static final String ACCESS_KEY = "testkey";
private static final String SECRET_KEY = "testsecret";
@TestHTTPResource
URI baseUri;
private final HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build();
@Test
public void uploadWithTagging() throws Exception
{
HttpResponse<Void> createBucket = send("PUT", baseUri.resolve(BUCKET), new byte[0], null, null);
assertEquals(200, createBucket.statusCode());
byte[] content = "hello tagging test".getBytes(StandardCharsets.UTF_8);
HttpResponse<Void> put = send("PUT", baseUri.resolve(BUCKET + "/" + KEY), content, "text/plain",
"project=r9t&owner=andreas");
assertEquals(200, put.statusCode());
assertTrue(put.headers().firstValue("ETag").isPresent());
HttpResponse<byte[]> tagging = sendForBytes("GET", baseUri.resolve(BUCKET + "/" + KEY + "?tagging="));
assertEquals(200, tagging.statusCode());
String xml = new String(tagging.body(), StandardCharsets.UTF_8);
assertTrue(xml.contains("<Key>project</Key>"));
assertTrue(xml.contains("<Value>r9t</Value>"));
assertTrue(xml.contains("<Key>owner</Key>"));
assertTrue(xml.contains("<Value>andreas</Value>"));
}
@AfterEach
public void cleanup() throws Exception
{
send("DELETE", baseUri.resolve(BUCKET + "/" + KEY), new byte[0], null, null);
send("DELETE", baseUri.resolve(BUCKET), new byte[0], null, null);
}
private HttpResponse<Void> send(String method, URI uri, byte[] body, String contentType, String tagging) throws Exception
{
HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
.method(method, HttpRequest.BodyPublishers.ofByteArray(body));
if (contentType != null)
{
builder.header("Content-Type", contentType);
}
if (tagging != null)
{
builder.header("x-amz-tagging", tagging);
}
SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, body);
return client.send(builder.build(), HttpResponse.BodyHandlers.discarding());
}
private HttpResponse<byte[]> sendForBytes(String method, URI uri) throws Exception
{
HttpRequest.Builder builder = HttpRequest.newBuilder(uri)
.method(method, HttpRequest.BodyPublishers.ofByteArray(new byte[0]));
SigV4TestSigner.sign(builder, method, uri, ACCESS_KEY, SECRET_KEY, new byte[0]);
return client.send(builder.build(), HttpResponse.BodyHandlers.ofByteArray());
}
}
@@ -1,3 +1,5 @@
config_ordinal=1000
dev.dinauer.r9t.workdir=target/test-r9t dev.dinauer.r9t.workdir=target/test-r9t
dev.dinauer.r9t.credentials.access-key=testkey dev.dinauer.r9t.credentials.access-key=testkey
dev.dinauer.r9t.credentials.secret-key=testsecret dev.dinauer.r9t.credentials.secret-key=testsecret