From a5bf4660d67aadf2ccb675dfa6ad035e1830d877 Mon Sep 17 00:00:00 2001 From: Andreas Dinauer Date: Sat, 10 Oct 2026 20:28:46 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Add=20Raft=20clustering=20with=20re?= =?UTF-8?q?plicated=20metadata=20and=20blobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- README.md | 31 + pom.xml | 16 + src/main/docker/Dockerfile.jvm | 2 +- .../dinauer/r9t/core/audit/AuditLogRepo.java | 53 +- .../r9t/core/audit/AuditLogService.java | 16 +- .../dinauer/r9t/core/auth/InternalToken.java | 30 + .../java/dev/dinauer/r9t/core/blob/Blob.java | 5 + .../dev/dinauer/r9t/core/blob/BlobClient.java | 48 ++ .../dinauer/r9t/core/blob/BlobReplicator.java | 157 +++++ .../dev/dinauer/r9t/core/blob/BlobRepo.java | 187 ++++++ .../dinauer/r9t/core/blob/BlobResource.java | 28 + .../core/blob/BlobSequenceInputStream.java | 70 +++ .../dinauer/r9t/core/blob/BlobService.java | 136 +++++ .../dev/dinauer/r9t/core/blob/StoredBlob.java | 5 + .../dinauer/r9t/core/bucket/BucketMapper.java | 19 + .../r9t/core/bucket/BucketMetadata.java | 14 + .../dinauer/r9t/core/bucket/BucketRepo.java | 181 +----- .../r9t/core/bucket/BucketService.java | 36 +- .../dinauer/r9t/core/init/InitService.java | 66 +- .../r9t/core/metadata/ApplyResult.java | 38 ++ .../dinauer/r9t/core/metadata/Command.java | 61 ++ .../r9t/core/metadata/CommandCodec.java | 177 ++++++ .../r9t/core/metadata/MetadataReader.java | 87 +++ .../r9t/core/metadata/MetadataStore.java | 570 ++++++++++++++++++ .../r9t/core/metadata/MetadataWriter.java | 90 +++ .../dinauer/r9t/core/metadata/Proposal.java | 7 + .../r9t/core/multipart/MultipartPart.java | 5 - .../core/multipart/MultipartUploadMapper.java | 20 + .../core/multipart/MultipartUploadRepo.java | 150 +---- .../multipart/MultipartUploadService.java | 112 ++-- .../r9t/core/multipart/PartMetadata.java | 7 + .../r9t/core/multipart/UploadMetadata.java | 26 + .../r9t/core/object/ObjectContent.java | 7 + .../r9t/core/object/ObjectMetadata.java | 15 + .../dinauer/r9t/core/object/ObjectRepo.java | 253 +------- .../r9t/core/object/ObjectResource.java | 12 +- .../r9t/core/object/ObjectService.java | 115 +++- .../r9t/core/object/S3ObjectMapper.java | 19 + .../r9t/core/raft/MetadataStateMachine.java | 151 +++++ .../dev/dinauer/r9t/core/raft/RaftNode.java | 21 + .../dinauer/r9t/core/raft/RaftService.java | 392 ++++++++++++ .../dev/dinauer/r9t/core/s3/DigestCopy.java | 18 +- .../java/dev/dinauer/r9t/core/s3/Digests.java | 5 + .../dinauer/r9t/core/s3/FileAttributes.java | 46 -- .../dev/dinauer/r9t/core/s3/S3ErrorCode.java | 2 + src/main/resources/application.properties | 3 +- .../r9t/core/metadata/CommandCodecTest.java | 50 ++ .../r9t/core/metadata/MetadataStoreTest.java | 164 +++++ src/test/resources/application.properties | 2 + 49 files changed, 2999 insertions(+), 726 deletions(-) create mode 100644 src/main/java/dev/dinauer/r9t/core/auth/InternalToken.java create mode 100644 src/main/java/dev/dinauer/r9t/core/blob/Blob.java create mode 100644 src/main/java/dev/dinauer/r9t/core/blob/BlobClient.java create mode 100644 src/main/java/dev/dinauer/r9t/core/blob/BlobReplicator.java create mode 100644 src/main/java/dev/dinauer/r9t/core/blob/BlobRepo.java create mode 100644 src/main/java/dev/dinauer/r9t/core/blob/BlobResource.java create mode 100644 src/main/java/dev/dinauer/r9t/core/blob/BlobSequenceInputStream.java create mode 100644 src/main/java/dev/dinauer/r9t/core/blob/BlobService.java create mode 100644 src/main/java/dev/dinauer/r9t/core/blob/StoredBlob.java create mode 100644 src/main/java/dev/dinauer/r9t/core/bucket/BucketMapper.java create mode 100644 src/main/java/dev/dinauer/r9t/core/bucket/BucketMetadata.java create mode 100644 src/main/java/dev/dinauer/r9t/core/metadata/ApplyResult.java create mode 100644 src/main/java/dev/dinauer/r9t/core/metadata/Command.java create mode 100644 src/main/java/dev/dinauer/r9t/core/metadata/CommandCodec.java create mode 100644 src/main/java/dev/dinauer/r9t/core/metadata/MetadataReader.java create mode 100644 src/main/java/dev/dinauer/r9t/core/metadata/MetadataStore.java create mode 100644 src/main/java/dev/dinauer/r9t/core/metadata/MetadataWriter.java create mode 100644 src/main/java/dev/dinauer/r9t/core/metadata/Proposal.java delete mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/MultipartPart.java create mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadMapper.java create mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/PartMetadata.java create mode 100644 src/main/java/dev/dinauer/r9t/core/multipart/UploadMetadata.java create mode 100644 src/main/java/dev/dinauer/r9t/core/object/ObjectContent.java create mode 100644 src/main/java/dev/dinauer/r9t/core/object/ObjectMetadata.java create mode 100644 src/main/java/dev/dinauer/r9t/core/object/S3ObjectMapper.java create mode 100644 src/main/java/dev/dinauer/r9t/core/raft/MetadataStateMachine.java create mode 100644 src/main/java/dev/dinauer/r9t/core/raft/RaftNode.java create mode 100644 src/main/java/dev/dinauer/r9t/core/raft/RaftService.java create mode 100644 src/main/java/dev/dinauer/r9t/core/s3/Digests.java delete mode 100644 src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java create mode 100644 src/test/java/dev/dinauer/r9t/core/metadata/CommandCodecTest.java create mode 100644 src/test/java/dev/dinauer/r9t/core/metadata/MetadataStoreTest.java diff --git a/README.md b/README.md index 24009b4..5adee78 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,36 @@ # core +## Clustering + +R9T replicates its metadata (buckets, objects, tags, multipart uploads) with Raft (Apache Ratis). +Object data is stored as immutable blobs under `/.blobs`; every node pulls the blobs it is missing +from the other nodes. + +| Variable | Default | Description | +|---|---|---| +| `DEV_DINAUER_R9T_RAFT_NODES` | empty | Comma separated `host:raftPort` list of all nodes. Empty runs a silent single node cluster. | +| `DEV_DINAUER_R9T_RAFT_SELF` | hostname | The entry of `DEV_DINAUER_R9T_RAFT_NODES` that is this node (host or `host:port`). | +| `DEV_DINAUER_R9T_RAFT_PORT` | `9870` | Raft port of a single node cluster (bound to `127.0.0.1`). | +| `DEV_DINAUER_R9T_RAFT_ADVERTISED_URL` | `http://:` | URL other nodes use to fetch blobs from this node. | +| `DEV_DINAUER_R9T_RAFT_MIN_REPLICAS` | `2` | A PUT returns once this many nodes hold the blob (capped at the cluster size). | +| `DEV_DINAUER_R9T_RAFT_REPLICATION_TIMEOUT` | `PT60S` | How long a PUT waits for `MIN_REPLICAS` before answering `503`. | +| `DEV_DINAUER_R9T_RAFT_TIMEOUT` | `PT15S` | Timeout for metadata writes and linearizable reads. | +| `DEV_DINAUER_R9T_RAFT_RECONCILE_INTERVAL` | `PT1M` | Interval of the background job that pulls missing blobs and deletes orphans. | + +Example for three nodes: + +```shell script +DEV_DINAUER_R9T_RAFT_NODES=r9t-1:9870,r9t-2:9870,r9t-3:9870 +DEV_DINAUER_R9T_RAFT_SELF=r9t-1:9870 +``` + +All nodes must share the same credentials, the HTTP port and the Raft port must be reachable between nodes. +The node list is fixed when a cluster is first started: a node refuses to start if the configured list differs from +the membership stored in `/.raft`. Turning an existing single node into a cluster is not supported yet. + +Data written by versions before clustering (one directory per bucket with xattr metadata) is not read anymore, +start with an empty workdir. + This project uses Quarkus, the Supersonic Subatomic Java Framework. If you want to learn more about Quarkus, please visit its website: . diff --git a/pom.xml b/pom.xml index f454c2c..4f4999f 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ 3.38.0 true 3.5.6 + 3.3.1 @@ -49,6 +50,21 @@ io.quarkus quarkus-arc + + org.apache.ratis + ratis-server + ${ratis.version} + + + org.apache.ratis + ratis-grpc + ${ratis.version} + + + org.apache.ratis + ratis-metrics-default + ${ratis.version} + io.quarkus quarkus-junit diff --git a/src/main/docker/Dockerfile.jvm b/src/main/docker/Dockerfile.jvm index 54c218f..22902e2 100644 --- a/src/main/docker/Dockerfile.jvm +++ b/src/main/docker/Dockerfile.jvm @@ -94,7 +94,7 @@ COPY --chown=185 target/quarkus-app/quarkus/ /deployments/quarkus/ USER root RUN mkdir -p /var/lib/r9t && chown 185 /var/lib/r9t -EXPOSE 8080 +EXPOSE 8080 9870 USER 185 ENV JAVA_OPTS_APPEND="-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager" ENV JAVA_APP_JAR="/deployments/quarkus-run.jar" diff --git a/src/main/java/dev/dinauer/r9t/core/audit/AuditLogRepo.java b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogRepo.java index 3bd611b..b19c1f3 100644 --- a/src/main/java/dev/dinauer/r9t/core/audit/AuditLogRepo.java +++ b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogRepo.java @@ -18,14 +18,34 @@ public class AuditLogRepo @ConfigProperty(name = "dev.dinauer.r9t.workdir", defaultValue = "/var/lib/r9t") String workdir; - public void append(AuditLogEntry entry) + private long lastIndex = -1; + + public synchronized void append(AuditLogEntry entry, long index) { - Path file = logFile(); try { - Files.createDirectories(file.getParent()); - Files.writeString(file, toLine(entry) + System.lineSeparator(), StandardOpenOption.CREATE, - StandardOpenOption.APPEND); + if (index > lastIndex()) + { + Path file = logFile(); + Files.createDirectories(file.getParent()); + Files.writeString(file, toLine(entry) + System.lineSeparator(), StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + Files.writeString(indexFile(), Long.toString(index)); + lastIndex = index; + } + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, "audit-log", e); + } + } + + public synchronized void resetIndex() + { + try + { + Files.deleteIfExists(indexFile()); + lastIndex = 0; } catch (IOException e) { @@ -53,11 +73,34 @@ public class AuditLogRepo } } + private long lastIndex() throws IOException + { + if (lastIndex < 0) + { + lastIndex = readIndex(); + } + return lastIndex; + } + + private long readIndex() throws IOException + { + if (Files.exists(indexFile())) + { + return Long.parseLong(Files.readString(indexFile()).trim()); + } + return 0; + } + private Path logFile() { return Path.of(workdir, ".audit", "audit.log"); } + private Path indexFile() + { + return Path.of(workdir, ".audit", "audit.index"); + } + private String toLine(AuditLogEntry entry) { return String.join("\t", diff --git a/src/main/java/dev/dinauer/r9t/core/audit/AuditLogService.java b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogService.java index 58e1624..1b67133 100644 --- a/src/main/java/dev/dinauer/r9t/core/audit/AuditLogService.java +++ b/src/main/java/dev/dinauer/r9t/core/audit/AuditLogService.java @@ -1,10 +1,10 @@ package dev.dinauer.r9t.core.audit; +import io.quarkus.arc.Arc; import io.quarkus.security.identity.SecurityIdentity; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import java.time.Instant; import java.util.Comparator; import java.util.List; @@ -17,10 +17,14 @@ public class AuditLogService @Inject SecurityIdentity securityIdentity; - public void record(AuditAction action, AuditResourceType resourceType, String bucket, String key) + public void record(AuditLogEntry entry, long index) { - AuditLogEntry entry = new AuditLogEntry(Instant.now(), action, resourceType, bucket, key, principal()); - auditLogRepo.append(entry); + auditLogRepo.append(entry, index); + } + + public void resetIndex() + { + auditLogRepo.resetIndex(); } public List list() @@ -30,9 +34,9 @@ public class AuditLogService .toList(); } - private String principal() + public String principal() { - if (securityIdentity.getPrincipal() != null) + if (Arc.container().requestContext().isActive() && securityIdentity.getPrincipal() != null) { return securityIdentity.getPrincipal().getName(); } diff --git a/src/main/java/dev/dinauer/r9t/core/auth/InternalToken.java b/src/main/java/dev/dinauer/r9t/core/auth/InternalToken.java new file mode 100644 index 0000000..3416f72 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/auth/InternalToken.java @@ -0,0 +1,30 @@ +package dev.dinauer.r9t.core.auth; + +import jakarta.enterprise.context.ApplicationScoped; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; + +@ApplicationScoped +public class InternalToken +{ + public static final String HEADER = "X-R9T-Internal-Token"; + + @ConfigProperty(name = "dev.dinauer.r9t.credentials.secret-key") + String secretKey; + + public String value() + { + return SigV4Signer.hex(SigV4Signer.hmacSha256(secretKey.getBytes(StandardCharsets.UTF_8), "r9t-internal")); + } + + public boolean matches(String candidate) + { + if (candidate == null) + { + return false; + } + return MessageDigest.isEqual(value().getBytes(StandardCharsets.UTF_8), candidate.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/blob/Blob.java b/src/main/java/dev/dinauer/r9t/core/blob/Blob.java new file mode 100644 index 0000000..8cdf3bc --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/blob/Blob.java @@ -0,0 +1,5 @@ +package dev.dinauer.r9t.core.blob; + +public record Blob(String id, long size, String sha256) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/blob/BlobClient.java b/src/main/java/dev/dinauer/r9t/core/blob/BlobClient.java new file mode 100644 index 0000000..26eeb12 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/blob/BlobClient.java @@ -0,0 +1,48 @@ +package dev.dinauer.r9t.core.blob; + +import dev.dinauer.r9t.core.auth.InternalToken; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; + +@ApplicationScoped +public class BlobClient +{ + private final HttpClient client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .connectTimeout(Duration.ofSeconds(5)) + .build(); + + @Inject + InternalToken internalToken; + + public InputStream open(String url, String blobId) throws IOException + { + HttpRequest request = HttpRequest.newBuilder(URI.create(url + "/_internal/blobs/" + blobId)) + .header(InternalToken.HEADER, internalToken.value()) + .GET() + .build(); + try + { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + if (response.statusCode() == 200) + { + return response.body(); + } + response.body().close(); + throw new IOException("Fetching blob " + blobId + " from " + url + " failed with status " + response.statusCode()); + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while fetching blob " + blobId + " from " + url, e); + } + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/blob/BlobReplicator.java b/src/main/java/dev/dinauer/r9t/core/blob/BlobReplicator.java new file mode 100644 index 0000000..cea9036 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/blob/BlobReplicator.java @@ -0,0 +1,157 @@ +package dev.dinauer.r9t.core.blob; + +import dev.dinauer.r9t.core.metadata.Command; +import dev.dinauer.r9t.core.metadata.MetadataStore; +import dev.dinauer.r9t.core.raft.RaftService; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.Instant; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +@ApplicationScoped +public class BlobReplicator +{ + private static final Logger LOG = Logger.getLogger(BlobReplicator.class); + private static final Duration ORPHAN_AGE = Duration.ofMinutes(10); + + @ConfigProperty(name = "dev.dinauer.r9t.raft.reconcile-interval", defaultValue = "PT1M") + Duration reconcileInterval; + + @ConfigProperty(name = "dev.dinauer.r9t.raft.replication-threads", defaultValue = "2") + int threads; + + @Inject + MetadataStore metadataStore; + + @Inject + BlobRepo blobRepo; + + @Inject + BlobClient blobClient; + + @Inject + RaftService raftService; + + private final Set pending = ConcurrentHashMap.newKeySet(); + private ExecutorService workers; + private ScheduledExecutorService scheduler; + + @PostConstruct + void init() + { + workers = Executors.newFixedThreadPool(threads); + scheduler = Executors.newSingleThreadScheduledExecutor(); + } + + @PreDestroy + void destroy() + { + scheduler.shutdownNow(); + workers.shutdownNow(); + } + + public void start() + { + scheduler.scheduleWithFixedDelay(this::reconcile, 0, reconcileInterval.toMillis(), TimeUnit.MILLISECONDS); + } + + public void schedule(Blob blob) + { + if (pending.add(blob.id())) + { + workers.submit(() -> replicate(blob)); + } + } + + private void replicate(Blob blob) + { + try + { + if (metadataStore.referenced(blob.id()) && !blobRepo.exists(blob.id())) + { + fetch(blob); + } + if (blobRepo.exists(blob.id()) && !metadataStore.holders(blob.id()).contains(raftService.selfId())) + { + raftService.submit(new Command.BlobStored(blob.id(), raftService.selfId()), blob.id()); + } + } + catch (RuntimeException | IOException e) + { + LOG.warnf(e, "Replicating blob %s failed, retrying on next reconcile", blob.id()); + } + finally + { + pending.remove(blob.id()); + } + } + + private void fetch(Blob blob) throws IOException + { + Map nodes = metadataStore.nodes(); + IOException failure = new IOException("No node could provide blob " + blob.id()); + boolean fetched = false; + for (String holder : metadataStore.holders(blob.id())) + { + String url = nodes.get(holder); + if (!fetched && url != null && !holder.equals(raftService.selfId())) + { + try (InputStream data = blobClient.open(url, blob.id())) + { + blobRepo.receive(blob, data); + fetched = true; + LOG.debugf("Fetched blob %s from %s", blob.id(), holder); + } + catch (IOException e) + { + failure.addSuppressed(e); + } + } + } + if (!fetched) + { + throw failure; + } + } + + private void reconcile() + { + try + { + raftService.barrier(); + metadataStore.objectBlobs().forEach(this::schedule); + sweep(); + } + catch (RuntimeException e) + { + LOG.warnf(e, "Blob reconciliation failed"); + } + } + + private void sweep() + { + Instant threshold = Instant.now().minus(ORPHAN_AGE); + for (String id : blobRepo.ids()) + { + if (!metadataStore.referenced(id) && blobRepo.lastModified(id).isBefore(threshold)) + { + LOG.infof("Deleting orphaned blob %s", id); + blobRepo.delete(id); + } + } + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/blob/BlobRepo.java b/src/main/java/dev/dinauer/r9t/core/blob/BlobRepo.java new file mode 100644 index 0000000..8db0ee9 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/blob/BlobRepo.java @@ -0,0 +1,187 @@ +package dev.dinauer.r9t.core.blob; + +import dev.dinauer.r9t.core.s3.DigestCopy; +import dev.dinauer.r9t.core.s3.Digests; +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.S3Exception; +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.ApplicationScoped; +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +@ApplicationScoped +public class BlobRepo +{ + private static final Pattern ID_PATTERN = Pattern.compile("^[0-9a-f]{32}$"); + + @ConfigProperty(name = "dev.dinauer.r9t.workdir", defaultValue = "/var/lib/r9t") + String workdir; + + @PostConstruct + void init() + { + try + { + Files.createDirectories(dataRoot()); + Files.createDirectories(tmpRoot()); + try (Stream leftovers = Files.list(tmpRoot())) + { + for (Path leftover : leftovers.toList()) + { + Files.delete(leftover); + } + } + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, tmpRoot().toString(), e); + } + } + + public StoredBlob store(InputStream data) + { + Path tmp = tmpFile(); + try + { + Digests digests = DigestCopy.copy(data, tmp); + Blob blob = new Blob(newId(), digests.size(), digests.sha256()); + moveIntoPlace(tmp, blob.id()); + return new StoredBlob(blob, digests.md5()); + } + catch (IOException e) + { + deleteQuietly(tmp, e); + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, "blob", e); + } + } + + public void receive(Blob expected, InputStream data) throws IOException + { + Path tmp = tmpFile(); + try + { + Digests digests = DigestCopy.copy(data, tmp); + if (digests.size() != expected.size() || !digests.sha256().equals(expected.sha256())) + { + throw new IOException("Blob " + expected.id() + " failed verification: expected " + expected.size() + + " bytes/" + expected.sha256() + " but got " + digests.size() + " bytes/" + digests.sha256()); + } + moveIntoPlace(tmp, expected.id()); + } + catch (IOException e) + { + deleteQuietly(tmp, e); + throw e; + } + } + + public boolean exists(String id) + { + return Files.isRegularFile(path(id)); + } + + public InputStream open(String id) throws IOException + { + return Files.newInputStream(path(id)); + } + + public Path path(String id) + { + if (!ID_PATTERN.matcher(id).matches()) + { + throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, id); + } + return dataRoot().resolve(id.substring(0, 2)).resolve(id.substring(2, 4)).resolve(id); + } + + public void delete(String id) + { + try + { + Files.deleteIfExists(path(id)); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, id, e); + } + } + + public List ids() + { + try (Stream files = Files.walk(dataRoot())) + { + return files + .filter(Files::isRegularFile) + .map(file -> file.getFileName().toString()) + .filter(name -> ID_PATTERN.matcher(name).matches()) + .sorted(Comparator.naturalOrder()) + .toList(); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, dataRoot().toString(), e); + } + } + + public Instant lastModified(String id) + { + try + { + return Files.getLastModifiedTime(path(id)).toInstant(); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, id, e); + } + } + + private void moveIntoPlace(Path tmp, String id) throws IOException + { + Path target = path(id); + Files.createDirectories(target.getParent()); + Files.move(tmp, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + + private void deleteQuietly(Path tmp, IOException cause) + { + try + { + Files.deleteIfExists(tmp); + } + catch (IOException e) + { + cause.addSuppressed(e); + } + } + + private Path tmpFile() + { + return tmpRoot().resolve(newId()); + } + + private String newId() + { + return UUID.randomUUID().toString().replace("-", ""); + } + + private Path dataRoot() + { + return Path.of(workdir, ".blobs", "data"); + } + + private Path tmpRoot() + { + return Path.of(workdir, ".blobs", "tmp"); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/blob/BlobResource.java b/src/main/java/dev/dinauer/r9t/core/blob/BlobResource.java new file mode 100644 index 0000000..6843fef --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/blob/BlobResource.java @@ -0,0 +1,28 @@ +package dev.dinauer.r9t.core.blob; + +import dev.dinauer.r9t.core.auth.InternalToken; +import jakarta.annotation.security.PermitAll; +import jakarta.inject.Inject; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.HeaderParam; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; + +@Path("/_internal/blobs") +@PermitAll +public class BlobResource +{ + @Inject + BlobService blobService; + + @GET + @Path("{id}") + @Produces(MediaType.APPLICATION_OCTET_STREAM) + public Response get(@PathParam("id") String id, @HeaderParam(InternalToken.HEADER) String token) + { + return Response.ok(blobService.local(id, token).toFile()).build(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/blob/BlobSequenceInputStream.java b/src/main/java/dev/dinauer/r9t/core/blob/BlobSequenceInputStream.java new file mode 100644 index 0000000..3c1127b --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/blob/BlobSequenceInputStream.java @@ -0,0 +1,70 @@ +package dev.dinauer.r9t.core.blob; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; + +public class BlobSequenceInputStream extends InputStream +{ + private final Iterator blobs; + private final Opener opener; + private InputStream current; + + public BlobSequenceInputStream(Iterator blobs, Opener opener) + { + this.blobs = blobs; + this.opener = opener; + } + + @Override + public int read() throws IOException + { + byte[] single = new byte[1]; + int count = read(single, 0, 1); + if (count == -1) + { + return -1; + } + return single[0] & 0xFF; + } + + @Override + public int read(byte[] buffer, int offset, int length) throws IOException + { + int count = -1; + while (count == -1 && advance()) + { + count = current.read(buffer, offset, length); + if (count == -1) + { + current.close(); + current = null; + } + } + return count; + } + + @Override + public void close() throws IOException + { + if (current != null) + { + current.close(); + current = null; + } + } + + private boolean advance() throws IOException + { + if (current == null && blobs.hasNext()) + { + current = opener.open(blobs.next()); + } + return current != null; + } + + public interface Opener + { + InputStream open(Blob blob) throws IOException; + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/blob/BlobService.java b/src/main/java/dev/dinauer/r9t/core/blob/BlobService.java new file mode 100644 index 0000000..ee0337f --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/blob/BlobService.java @@ -0,0 +1,136 @@ +package dev.dinauer.r9t.core.blob; + +import dev.dinauer.r9t.core.auth.InternalToken; +import dev.dinauer.r9t.core.metadata.MetadataStore; +import dev.dinauer.r9t.core.raft.RaftService; +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.S3Exception; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Map; + +@ApplicationScoped +public class BlobService +{ + private static final Logger LOG = Logger.getLogger(BlobService.class); + + @ConfigProperty(name = "dev.dinauer.r9t.raft.min-replicas", defaultValue = "2") + int minReplicas; + + @ConfigProperty(name = "dev.dinauer.r9t.raft.replication-timeout", defaultValue = "PT60S") + Duration replicationTimeout; + + @Inject + BlobRepo blobRepo; + + @Inject + BlobClient blobClient; + + @Inject + MetadataStore metadataStore; + + @Inject + RaftService raftService; + + @Inject + InternalToken internalToken; + + public StoredBlob store(InputStream data) + { + return blobRepo.store(data); + } + + public StoredBlob assemble(List parts) + { + try (InputStream assembled = new BlobSequenceInputStream(parts.iterator(), this::openStream)) + { + return blobRepo.store(assembled); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, "assemble", e); + } + } + + public void commit(Blob blob, Runnable submit) + { + try + { + submit.run(); + } + catch (RuntimeException e) + { + blobRepo.delete(blob.id()); + throw e; + } + } + + public void awaitReplicas(Blob blob, String resource) + { + int required = Math.min(minReplicas, raftService.clusterSize()); + if (!metadataStore.awaitHolders(blob.id(), required, replicationTimeout)) + { + throw new S3Exception(S3ErrorCode.SERVICE_UNAVAILABLE, resource); + } + } + + public InputStream open(Blob blob) + { + try + { + return openStream(blob); + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.SERVICE_UNAVAILABLE, blob.id(), e); + } + } + + public Path local(String id, String token) + { + if (!internalToken.matches(token)) + { + throw new S3Exception(S3ErrorCode.ACCESS_DENIED, id); + } + if (!blobRepo.exists(id)) + { + throw new S3Exception(S3ErrorCode.NO_SUCH_KEY, id); + } + return blobRepo.path(id); + } + + private InputStream openStream(Blob blob) throws IOException + { + if (blobRepo.exists(blob.id())) + { + return blobRepo.open(blob.id()); + } + Map nodes = metadataStore.nodes(); + IOException failure = new IOException("Blob " + blob.id() + " is not available on any node"); + for (String holder : metadataStore.holders(blob.id())) + { + String url = nodes.get(holder); + if (url != null && !holder.equals(raftService.selfId())) + { + try + { + return blobClient.open(url, blob.id()); + } + catch (IOException e) + { + LOG.warnf("Could not read blob %s from %s: %s", blob.id(), holder, e.getMessage()); + failure.addSuppressed(e); + } + } + } + throw failure; + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/blob/StoredBlob.java b/src/main/java/dev/dinauer/r9t/core/blob/StoredBlob.java new file mode 100644 index 0000000..5aa6164 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/blob/StoredBlob.java @@ -0,0 +1,5 @@ +package dev.dinauer.r9t.core.blob; + +public record StoredBlob(Blob blob, String md5) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketMapper.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketMapper.java new file mode 100644 index 0000000..c13feac --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketMapper.java @@ -0,0 +1,19 @@ +package dev.dinauer.r9t.core.bucket; + +import jakarta.enterprise.context.ApplicationScoped; + +import java.util.List; + +@ApplicationScoped +public class BucketMapper +{ + public Bucket map(BucketMetadata input) + { + return new Bucket(input.name(), input.creationDate()); + } + + public List map(List input) + { + return input.stream().map(this::map).toList(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketMetadata.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketMetadata.java new file mode 100644 index 0000000..e9e1fb8 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketMetadata.java @@ -0,0 +1,14 @@ +package dev.dinauer.r9t.core.bucket; + +import dev.dinauer.r9t.core.s3.Tag; + +import java.time.Instant; +import java.util.List; + +public record BucketMetadata(String name, Instant creationDate, List tags) +{ + public BucketMetadata withTags(List newTags) + { + return new BucketMetadata(name, creationDate, List.copyOf(newTags)); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java index 9175fe1..a24f9b9 100644 --- a/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketRepo.java @@ -1,189 +1,60 @@ package dev.dinauer.r9t.core.bucket; -import dev.dinauer.r9t.core.s3.FileAttributes; -import dev.dinauer.r9t.core.s3.S3ErrorCode; -import dev.dinauer.r9t.core.s3.S3Exception; +import dev.dinauer.r9t.core.metadata.Command; +import dev.dinauer.r9t.core.metadata.MetadataStore; +import dev.dinauer.r9t.core.object.ObjectMetadata; +import dev.dinauer.r9t.core.raft.RaftService; import dev.dinauer.r9t.core.s3.Tag; -import dev.dinauer.r9t.core.s3.TagCodec; -import jakarta.annotation.PostConstruct; import jakarta.enterprise.context.ApplicationScoped; -import org.eclipse.microprofile.config.inject.ConfigProperty; +import jakarta.inject.Inject; -import java.io.IOException; -import java.nio.file.DirectoryNotEmptyException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; import java.util.List; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Stream; +import java.util.Optional; @ApplicationScoped public class BucketRepo { - @ConfigProperty(name = "dev.dinauer.r9t.workdir", defaultValue = "/var/lib/r9t") - String workdir; + @Inject + RaftService raftService; - @PostConstruct - void init() - { - try - { - Files.createDirectories(root()); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, workdir, e); - } - } + @Inject + MetadataStore metadataStore; - public Path root() + public Optional find(String bucket) { - return Path.of(workdir); - } - - public Path resolve(String bucket) - { - Path target = root().resolve(bucket).normalize(); - if (!target.startsWith(root())) - { - throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, bucket); - } - return target; + return raftService.read(() -> metadataStore.bucket(bucket)); } public boolean exists(String bucket) { - return Files.isDirectory(resolve(bucket)); + return find(bucket).isPresent(); } - public void create(String bucket) + public List list() { - try - { - Files.createDirectories(resolve(bucket)); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); - } + return raftService.read(metadataStore::buckets); + } + + public void create(String bucket, Instant creationDate) + { + raftService.submit(new Command.CreateBucket(bucket, creationDate), bucket); } public void delete(String bucket) { - try - { - Files.delete(resolve(bucket)); - } - catch (DirectoryNotEmptyException e) - { - throw new S3Exception(S3ErrorCode.BUCKET_NOT_EMPTY, bucket, e); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); - } - } - - public List list() - { - try (Stream entries = Files.list(root())) - { - return entries - .filter(Files::isDirectory) - .filter(path -> !path.getFileName().toString().startsWith(".")) - .map(this::toBucket) - .toList(); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, "/", e); - } - } - - public List getTags(String bucket) - { - Path target = resolve(bucket); - try - { - if (FileAttributes.exists(target, FileAttributes.TAGS)) - { - return TagCodec.parseQueryString(FileAttributes.read(target, FileAttributes.TAGS)); - } - return List.of(); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); - } + raftService.submit(new Command.DeleteBucket(bucket), bucket); } public void putTags(String bucket, List tags) { - Path target = resolve(bucket); - try - { - FileAttributes.write(target, FileAttributes.TAGS, TagCodec.toQueryString(tags)); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); - } - } - - public void deleteTags(String bucket) - { - Path target = resolve(bucket); - try - { - if (FileAttributes.exists(target, FileAttributes.TAGS)) - { - FileAttributes.delete(target, FileAttributes.TAGS); - } - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); - } + raftService.submit(new Command.PutBucketTags(bucket, tags), bucket); } public BucketStats stats(String bucket) { - Path bucketPath = resolve(bucket); - AtomicLong objectCount = new AtomicLong(); - AtomicLong totalSize = new AtomicLong(); - try - { - Files.walkFileTree(bucketPath, new SimpleFileVisitor<>() - { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) - { - objectCount.incrementAndGet(); - totalSize.addAndGet(attributes.size()); - return FileVisitResult.CONTINUE; - } - }); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); - } - return new BucketStats(bucket, objectCount.get(), totalSize.get()); - } - - private Bucket toBucket(Path dir) - { - try - { - BasicFileAttributes attributes = Files.readAttributes(dir, BasicFileAttributes.class); - return new Bucket(dir.getFileName().toString(), attributes.creationTime().toInstant()); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, dir.toString(), e); - } + List objects = raftService.read(() -> metadataStore.objects(bucket, null)); + long totalSize = objects.stream().mapToLong(object -> object.blob().size()).sum(); + return new BucketStats(bucket, objects.size(), totalSize); } } diff --git a/src/main/java/dev/dinauer/r9t/core/bucket/BucketService.java b/src/main/java/dev/dinauer/r9t/core/bucket/BucketService.java index 838bdf6..54bd1b0 100644 --- a/src/main/java/dev/dinauer/r9t/core/bucket/BucketService.java +++ b/src/main/java/dev/dinauer/r9t/core/bucket/BucketService.java @@ -1,8 +1,5 @@ package dev.dinauer.r9t.core.bucket; -import dev.dinauer.r9t.core.audit.AuditAction; -import dev.dinauer.r9t.core.audit.AuditLogService; -import dev.dinauer.r9t.core.audit.AuditResourceType; import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3Exception; import dev.dinauer.r9t.core.s3.Tag; @@ -10,6 +7,7 @@ import dev.dinauer.r9t.core.s3.TagValidator; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; +import java.time.Instant; import java.util.List; import java.util.regex.Pattern; @@ -22,11 +20,11 @@ public class BucketService BucketRepo bucketRepo; @Inject - AuditLogService auditLogService; + BucketMapper bucketMapper; public List list() { - return bucketRepo.list(); + return bucketMapper.map(bucketRepo.list()); } public List stats() @@ -40,48 +38,40 @@ public class BucketService } public void create(String name) + { + create(name, Instant.now()); + } + + public void create(String name, Instant creationDate) { validate(name); - if (bucketRepo.exists(name)) - { - throw new S3Exception(S3ErrorCode.BUCKET_ALREADY_EXISTS, name); - } - bucketRepo.create(name); - auditLogService.record(AuditAction.CREATE, AuditResourceType.BUCKET, name, null); + bucketRepo.create(name, creationDate); } public void delete(String name) { - require(name); bucketRepo.delete(name); - auditLogService.record(AuditAction.DELETE, AuditResourceType.BUCKET, name, null); } - public void require(String name) + public BucketMetadata require(String name) { - if (!bucketRepo.exists(name)) - { - throw new S3Exception(S3ErrorCode.NO_SUCH_BUCKET, name); - } + return bucketRepo.find(name).orElseThrow(() -> new S3Exception(S3ErrorCode.NO_SUCH_BUCKET, name)); } public List getTags(String name) { - require(name); - return bucketRepo.getTags(name); + return require(name).tags(); } public void putTags(String name, List tags) { - require(name); TagValidator.validate(tags); bucketRepo.putTags(name, tags); } public void deleteTags(String name) { - require(name); - bucketRepo.deleteTags(name); + bucketRepo.putTags(name, List.of()); } private void validate(String name) diff --git a/src/main/java/dev/dinauer/r9t/core/init/InitService.java b/src/main/java/dev/dinauer/r9t/core/init/InitService.java index d832c95..5560e7d 100644 --- a/src/main/java/dev/dinauer/r9t/core/init/InitService.java +++ b/src/main/java/dev/dinauer/r9t/core/init/InitService.java @@ -1,6 +1,11 @@ package dev.dinauer.r9t.core.init; +import dev.dinauer.r9t.core.blob.BlobReplicator; import dev.dinauer.r9t.core.bucket.BucketService; +import dev.dinauer.r9t.core.raft.RaftService; +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.S3Exception; +import io.quarkus.runtime.ShutdownEvent; import io.quarkus.runtime.StartupEvent; import jakarta.enterprise.context.ApplicationScoped; import jakarta.enterprise.event.Observes; @@ -10,6 +15,8 @@ import org.jboss.logging.Logger; import java.util.List; import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; @ApplicationScoped public class InitService @@ -19,10 +26,49 @@ public class InitService @ConfigProperty(name = "dev.dinauer.r9t.init.buckets") Optional> buckets; + @Inject + RaftService raftService; + @Inject BucketService bucketService; + @Inject + BlobReplicator blobReplicator; + + private ExecutorService executor; + void init(@Observes StartupEvent event) + { + executor = Executors.newSingleThreadExecutor(); + executor.submit(this::run); + } + + void stop(@Observes ShutdownEvent event) + { + executor.shutdownNow(); + } + + private void run() + { + try + { + if (!raftService.awaitReady()) + { + LOG.info("Node initialization aborted"); + return; + } + raftService.register(); + bootstrap(); + blobReplicator.start(); + LOG.info("Node initialization finished"); + } + catch (RuntimeException e) + { + LOG.error("Node initialization failed", e); + } + } + + private void bootstrap() { for (String bucket : buckets.orElse(List.of())) { @@ -32,9 +78,25 @@ public class InitService } else { - bucketService.create(bucket); - LOG.infof("Bootstrapped bucket '%s'", bucket); + create(bucket); } } } + + private void create(String bucket) + { + try + { + bucketService.create(bucket); + LOG.infof("Bootstrapped bucket '%s'", bucket); + } + catch (S3Exception e) + { + if (e.getCode() != S3ErrorCode.BUCKET_ALREADY_EXISTS) + { + throw e; + } + LOG.infof("Bootstrap bucket '%s' was created by another node, skipping", bucket); + } + } } diff --git a/src/main/java/dev/dinauer/r9t/core/metadata/ApplyResult.java b/src/main/java/dev/dinauer/r9t/core/metadata/ApplyResult.java new file mode 100644 index 0000000..0209e69 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/metadata/ApplyResult.java @@ -0,0 +1,38 @@ +package dev.dinauer.r9t.core.metadata; + +import dev.dinauer.r9t.core.blob.Blob; +import dev.dinauer.r9t.core.s3.S3ErrorCode; + +import java.util.List; + +public record ApplyResult(S3ErrorCode error, List added, List released) +{ + public static ApplyResult ok() + { + return new ApplyResult(null, List.of(), List.of()); + } + + public static ApplyResult released(List released) + { + return new ApplyResult(null, List.of(), List.copyOf(released)); + } + + public static ApplyResult error(S3ErrorCode error) + { + return new ApplyResult(error, List.of(), List.of()); + } + + public boolean success() + { + return error == null; + } + + public String code() + { + if (error == null) + { + return ""; + } + return error.name(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/metadata/Command.java b/src/main/java/dev/dinauer/r9t/core/metadata/Command.java new file mode 100644 index 0000000..9cdea53 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/metadata/Command.java @@ -0,0 +1,61 @@ +package dev.dinauer.r9t.core.metadata; + +import dev.dinauer.r9t.core.blob.Blob; +import dev.dinauer.r9t.core.s3.Tag; + +import java.time.Instant; +import java.util.List; + +public sealed interface Command +{ + record RegisterNode(String nodeId, String url) implements Command + { + } + + record CreateBucket(String name, Instant creationDate) implements Command + { + } + + record DeleteBucket(String name) implements Command + { + } + + record PutBucketTags(String name, List tags) implements Command + { + } + + record PutObject(String bucket, String key, String etag, String contentType, List tags, Instant lastModified, + Blob blob, String nodeId) implements Command + { + } + + record DeleteObject(String bucket, String key) implements Command + { + } + + record PutObjectTags(String bucket, String key, List tags) implements Command + { + } + + record InitiateUpload(String uploadId, String bucket, String key, String contentType, List tags, + Instant initiated) implements Command + { + } + + record PutPart(String uploadId, int partNumber, String etag, Blob blob, String nodeId) implements Command + { + } + + record CompleteUpload(String uploadId, String bucket, String key, String etag, Instant lastModified, Blob blob, + String nodeId) implements Command + { + } + + record AbortUpload(String uploadId) implements Command + { + } + + record BlobStored(String blobId, String nodeId) implements Command + { + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/metadata/CommandCodec.java b/src/main/java/dev/dinauer/r9t/core/metadata/CommandCodec.java new file mode 100644 index 0000000..11cff76 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/metadata/CommandCodec.java @@ -0,0 +1,177 @@ +package dev.dinauer.r9t.core.metadata; + +import jakarta.enterprise.context.ApplicationScoped; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; + +@ApplicationScoped +public class CommandCodec +{ + private static final int REGISTER_NODE = 1; + private static final int CREATE_BUCKET = 2; + private static final int DELETE_BUCKET = 3; + private static final int PUT_BUCKET_TAGS = 4; + private static final int PUT_OBJECT = 5; + private static final int DELETE_OBJECT = 6; + private static final int PUT_OBJECT_TAGS = 7; + private static final int INITIATE_UPLOAD = 8; + private static final int PUT_PART = 9; + private static final int COMPLETE_UPLOAD = 10; + private static final int ABORT_UPLOAD = 11; + private static final int BLOB_STORED = 12; + + public byte[] encode(Proposal proposal) + { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + MetadataWriter writer = new MetadataWriter(bytes); + try + { + writer.writeInstant(proposal.timestamp()); + writer.writeNullableString(proposal.principal()); + write(writer, proposal.command()); + writer.flush(); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + return bytes.toByteArray(); + } + + public Proposal decode(byte[] data) + { + MetadataReader reader = new MetadataReader(new ByteArrayInputStream(data)); + try + { + return new Proposal(reader.readInstant(), reader.readNullableString(), read(reader)); + } + catch (IOException e) + { + throw new UncheckedIOException(e); + } + } + + private void write(MetadataWriter writer, Command command) throws IOException + { + switch (command) + { + case Command.RegisterNode c -> + { + writer.writeByte(REGISTER_NODE); + writer.writeString(c.nodeId()); + writer.writeString(c.url()); + } + case Command.CreateBucket c -> + { + writer.writeByte(CREATE_BUCKET); + writer.writeString(c.name()); + writer.writeInstant(c.creationDate()); + } + case Command.DeleteBucket c -> + { + writer.writeByte(DELETE_BUCKET); + writer.writeString(c.name()); + } + case Command.PutBucketTags c -> + { + writer.writeByte(PUT_BUCKET_TAGS); + writer.writeString(c.name()); + writer.writeTags(c.tags()); + } + case Command.PutObject c -> + { + writer.writeByte(PUT_OBJECT); + writer.writeString(c.bucket()); + writer.writeString(c.key()); + writer.writeString(c.etag()); + writer.writeString(c.contentType()); + writer.writeTags(c.tags()); + writer.writeInstant(c.lastModified()); + writer.writeBlob(c.blob()); + writer.writeString(c.nodeId()); + } + case Command.DeleteObject c -> + { + writer.writeByte(DELETE_OBJECT); + writer.writeString(c.bucket()); + writer.writeString(c.key()); + } + case Command.PutObjectTags c -> + { + writer.writeByte(PUT_OBJECT_TAGS); + writer.writeString(c.bucket()); + writer.writeString(c.key()); + writer.writeTags(c.tags()); + } + case Command.InitiateUpload c -> + { + writer.writeByte(INITIATE_UPLOAD); + writer.writeString(c.uploadId()); + writer.writeString(c.bucket()); + writer.writeString(c.key()); + writer.writeString(c.contentType()); + writer.writeTags(c.tags()); + writer.writeInstant(c.initiated()); + } + case Command.PutPart c -> + { + writer.writeByte(PUT_PART); + writer.writeString(c.uploadId()); + writer.writeInt(c.partNumber()); + writer.writeString(c.etag()); + writer.writeBlob(c.blob()); + writer.writeString(c.nodeId()); + } + case Command.CompleteUpload c -> + { + writer.writeByte(COMPLETE_UPLOAD); + writer.writeString(c.uploadId()); + writer.writeString(c.bucket()); + writer.writeString(c.key()); + writer.writeString(c.etag()); + writer.writeInstant(c.lastModified()); + writer.writeBlob(c.blob()); + writer.writeString(c.nodeId()); + } + case Command.AbortUpload c -> + { + writer.writeByte(ABORT_UPLOAD); + writer.writeString(c.uploadId()); + } + case Command.BlobStored c -> + { + writer.writeByte(BLOB_STORED); + writer.writeString(c.blobId()); + writer.writeString(c.nodeId()); + } + } + } + + private Command read(MetadataReader reader) throws IOException + { + int type = reader.readByte(); + return switch (type) + { + case REGISTER_NODE -> new Command.RegisterNode(reader.readString(), reader.readString()); + case CREATE_BUCKET -> new Command.CreateBucket(reader.readString(), reader.readInstant()); + case DELETE_BUCKET -> new Command.DeleteBucket(reader.readString()); + case PUT_BUCKET_TAGS -> new Command.PutBucketTags(reader.readString(), reader.readTags()); + case PUT_OBJECT -> new Command.PutObject(reader.readString(), reader.readString(), reader.readString(), + reader.readString(), reader.readTags(), reader.readInstant(), reader.readBlob(), reader.readString()); + case DELETE_OBJECT -> new Command.DeleteObject(reader.readString(), reader.readString()); + case PUT_OBJECT_TAGS -> new Command.PutObjectTags(reader.readString(), reader.readString(), reader.readTags()); + case INITIATE_UPLOAD -> new Command.InitiateUpload(reader.readString(), reader.readString(), + reader.readString(), reader.readString(), reader.readTags(), reader.readInstant()); + case PUT_PART -> new Command.PutPart(reader.readString(), reader.readInt(), reader.readString(), + reader.readBlob(), reader.readString()); + case COMPLETE_UPLOAD -> new Command.CompleteUpload(reader.readString(), reader.readString(), + reader.readString(), reader.readString(), reader.readInstant(), reader.readBlob(), reader.readString()); + case ABORT_UPLOAD -> new Command.AbortUpload(reader.readString()); + case BLOB_STORED -> new Command.BlobStored(reader.readString(), reader.readString()); + default -> throw new IOException("Unknown command type " + type); + }; + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/metadata/MetadataReader.java b/src/main/java/dev/dinauer/r9t/core/metadata/MetadataReader.java new file mode 100644 index 0000000..d307380 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/metadata/MetadataReader.java @@ -0,0 +1,87 @@ +package dev.dinauer.r9t.core.metadata; + +import dev.dinauer.r9t.core.blob.Blob; +import dev.dinauer.r9t.core.s3.Tag; + +import java.io.DataInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +public class MetadataReader +{ + private final DataInputStream input; + + public MetadataReader(InputStream input) + { + this.input = new DataInputStream(input); + } + + public int readByte() throws IOException + { + return input.readUnsignedByte(); + } + + public int readInt() throws IOException + { + return input.readInt(); + } + + public long readLong() throws IOException + { + return input.readLong(); + } + + public String readString() throws IOException + { + byte[] bytes = new byte[input.readInt()]; + input.readFully(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + public String readNullableString() throws IOException + { + if (input.readBoolean()) + { + return readString(); + } + return null; + } + + public List readStrings() throws IOException + { + int count = input.readInt(); + List values = new ArrayList<>(count); + for (int i = 0; i < count; i++) + { + values.add(readString()); + } + return values; + } + + public Instant readInstant() throws IOException + { + long seconds = input.readLong(); + int nanos = input.readInt(); + return Instant.ofEpochSecond(seconds, nanos); + } + + public List readTags() throws IOException + { + int count = input.readInt(); + List tags = new ArrayList<>(count); + for (int i = 0; i < count; i++) + { + tags.add(new Tag(readString(), readString())); + } + return List.copyOf(tags); + } + + public Blob readBlob() throws IOException + { + return new Blob(readString(), input.readLong(), readString()); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/metadata/MetadataStore.java b/src/main/java/dev/dinauer/r9t/core/metadata/MetadataStore.java new file mode 100644 index 0000000..8f9b5cb --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/metadata/MetadataStore.java @@ -0,0 +1,570 @@ +package dev.dinauer.r9t.core.metadata; + +import dev.dinauer.r9t.core.blob.Blob; +import dev.dinauer.r9t.core.bucket.BucketMetadata; +import dev.dinauer.r9t.core.multipart.PartMetadata; +import dev.dinauer.r9t.core.multipart.UploadMetadata; +import dev.dinauer.r9t.core.object.ObjectMetadata; +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.Tag; +import jakarta.inject.Singleton; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; + +@Singleton +public class MetadataStore +{ + private static final int SNAPSHOT_VERSION = 1; + + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final Object progress = new Object(); + private final Map nodes = new TreeMap<>(); + private final Map buckets = new TreeMap<>(); + private final Map> objects = new HashMap<>(); + private final Map uploads = new HashMap<>(); + private final Map> holders = new HashMap<>(); + private long appliedIndex; + + public ApplyResult apply(Command command, long index) + { + ApplyResult result = write(() -> + { + ApplyResult applied = dispatch(command); + appliedIndex = index; + return applied; + }); + signal(); + return result; + } + + public long appliedIndex() + { + return read(() -> appliedIndex); + } + + public Map nodes() + { + return read(() -> Map.copyOf(nodes)); + } + + public List buckets() + { + return read(() -> List.copyOf(buckets.values())); + } + + public Optional bucket(String name) + { + return read(() -> Optional.ofNullable(buckets.get(name))); + } + + public Optional object(String bucket, String key) + { + return read(() -> Optional.ofNullable(objects.getOrDefault(bucket, new TreeMap<>()).get(key))); + } + + public List objects(String bucket, String prefix) + { + return read(() -> + { + TreeMap bucketObjects = objects.getOrDefault(bucket, new TreeMap<>()); + if (prefix == null || prefix.isEmpty()) + { + return List.copyOf(bucketObjects.values()); + } + return bucketObjects.tailMap(prefix, true).values().stream() + .takeWhile(object -> object.key().startsWith(prefix)) + .toList(); + }); + } + + public Optional upload(String uploadId) + { + return read(() -> Optional.ofNullable(uploads.get(uploadId))); + } + + public Set holders(String blobId) + { + return read(() -> Set.copyOf(holders.getOrDefault(blobId, Set.of()))); + } + + public boolean referenced(String blobId) + { + return read(() -> holders.containsKey(blobId)); + } + + public List objectBlobs() + { + return read(() -> objects.values().stream() + .flatMap(bucketObjects -> bucketObjects.values().stream()) + .map(ObjectMetadata::blob) + .toList()); + } + + public boolean awaitApplied(long index, Duration timeout) + { + return await(() -> appliedIndex() >= index, timeout); + } + + public boolean awaitHolders(String blobId, int count, Duration timeout) + { + return await(() -> holders(blobId).size() >= count, timeout); + } + + public void writeSnapshot(OutputStream output) throws IOException + { + lock.readLock().lock(); + try + { + MetadataWriter writer = new MetadataWriter(output); + writer.writeInt(SNAPSHOT_VERSION); + writeNodes(writer); + writeBuckets(writer); + writeUploads(writer); + writeHolders(writer); + writer.flush(); + } + finally + { + lock.readLock().unlock(); + } + } + + public void readSnapshot(InputStream input, long index) throws IOException + { + lock.writeLock().lock(); + try + { + MetadataReader reader = new MetadataReader(input); + int version = reader.readInt(); + if (version != SNAPSHOT_VERSION) + { + throw new IOException("Unsupported metadata snapshot version " + version); + } + clear(); + readNodes(reader); + readBuckets(reader); + readUploads(reader); + readHolders(reader); + appliedIndex = index; + } + finally + { + lock.writeLock().unlock(); + } + signal(); + } + + private ApplyResult dispatch(Command command) + { + return switch (command) + { + case Command.RegisterNode c -> registerNode(c); + case Command.CreateBucket c -> createBucket(c); + case Command.DeleteBucket c -> deleteBucket(c); + case Command.PutBucketTags c -> putBucketTags(c); + case Command.PutObject c -> putObject(c); + case Command.DeleteObject c -> deleteObject(c); + case Command.PutObjectTags c -> putObjectTags(c); + case Command.InitiateUpload c -> initiateUpload(c); + case Command.PutPart c -> putPart(c); + case Command.CompleteUpload c -> completeUpload(c); + case Command.AbortUpload c -> abortUpload(c); + case Command.BlobStored c -> blobStored(c); + }; + } + + private ApplyResult registerNode(Command.RegisterNode command) + { + nodes.put(command.nodeId(), command.url()); + return ApplyResult.ok(); + } + + private ApplyResult createBucket(Command.CreateBucket command) + { + if (buckets.containsKey(command.name())) + { + return ApplyResult.error(S3ErrorCode.BUCKET_ALREADY_EXISTS); + } + buckets.put(command.name(), new BucketMetadata(command.name(), command.creationDate(), List.of())); + objects.put(command.name(), new TreeMap<>()); + return ApplyResult.ok(); + } + + private ApplyResult deleteBucket(Command.DeleteBucket command) + { + if (!buckets.containsKey(command.name())) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_BUCKET); + } + if (!objects.get(command.name()).isEmpty()) + { + return ApplyResult.error(S3ErrorCode.BUCKET_NOT_EMPTY); + } + buckets.remove(command.name()); + objects.remove(command.name()); + List uploadIds = uploads.values().stream() + .filter(upload -> upload.bucket().equals(command.name())) + .map(UploadMetadata::uploadId) + .toList(); + List released = new ArrayList<>(); + for (String uploadId : uploadIds) + { + released.addAll(removeUpload(uploadId)); + } + return ApplyResult.released(released); + } + + private ApplyResult putBucketTags(Command.PutBucketTags command) + { + BucketMetadata bucket = buckets.get(command.name()); + if (bucket == null) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_BUCKET); + } + buckets.put(command.name(), bucket.withTags(command.tags())); + return ApplyResult.ok(); + } + + private ApplyResult putObject(Command.PutObject command) + { + TreeMap bucketObjects = objects.get(command.bucket()); + if (bucketObjects == null) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_BUCKET); + } + ObjectMetadata object = new ObjectMetadata(command.key(), command.etag(), command.contentType(), + command.lastModified(), List.copyOf(command.tags()), command.blob()); + return storeObject(bucketObjects, object, command.nodeId(), List.of()); + } + + private ApplyResult deleteObject(Command.DeleteObject command) + { + TreeMap bucketObjects = objects.get(command.bucket()); + if (bucketObjects == null) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_BUCKET); + } + ObjectMetadata removed = bucketObjects.remove(command.key()); + if (removed != null) + { + return ApplyResult.released(List.of(release(removed.blob().id()))); + } + return ApplyResult.ok(); + } + + private ApplyResult putObjectTags(Command.PutObjectTags command) + { + TreeMap bucketObjects = objects.get(command.bucket()); + if (bucketObjects == null) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_BUCKET); + } + ObjectMetadata object = bucketObjects.get(command.key()); + if (object == null) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_KEY); + } + bucketObjects.put(command.key(), object.withTags(command.tags())); + return ApplyResult.ok(); + } + + private ApplyResult initiateUpload(Command.InitiateUpload command) + { + if (!buckets.containsKey(command.bucket())) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_BUCKET); + } + uploads.put(command.uploadId(), UploadMetadata.of(command.uploadId(), command.bucket(), command.key(), + command.contentType(), command.initiated(), command.tags(), Map.of())); + return ApplyResult.ok(); + } + + private ApplyResult putPart(Command.PutPart command) + { + UploadMetadata upload = uploads.get(command.uploadId()); + if (upload == null) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_UPLOAD); + } + PartMetadata previous = upload.parts().get(command.partNumber()); + uploads.put(command.uploadId(), upload.withPart(new PartMetadata(command.partNumber(), command.etag(), command.blob()))); + holders.put(command.blob().id(), new TreeSet<>(Set.of(command.nodeId()))); + if (previous != null) + { + return ApplyResult.released(List.of(release(previous.blob().id()))); + } + return ApplyResult.ok(); + } + + private ApplyResult completeUpload(Command.CompleteUpload command) + { + UploadMetadata upload = uploads.get(command.uploadId()); + if (upload == null || !upload.bucket().equals(command.bucket()) || !upload.key().equals(command.key())) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_UPLOAD); + } + TreeMap bucketObjects = objects.get(upload.bucket()); + if (bucketObjects == null) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_BUCKET); + } + List releasedParts = removeUpload(command.uploadId()); + ObjectMetadata object = new ObjectMetadata(upload.key(), command.etag(), upload.contentType(), + command.lastModified(), upload.tags(), command.blob()); + return storeObject(bucketObjects, object, command.nodeId(), releasedParts); + } + + private ApplyResult abortUpload(Command.AbortUpload command) + { + if (!uploads.containsKey(command.uploadId())) + { + return ApplyResult.error(S3ErrorCode.NO_SUCH_UPLOAD); + } + return ApplyResult.released(removeUpload(command.uploadId())); + } + + private ApplyResult blobStored(Command.BlobStored command) + { + Set blobHolders = holders.get(command.blobId()); + if (blobHolders != null) + { + blobHolders.add(command.nodeId()); + return ApplyResult.ok(); + } + return ApplyResult.released(List.of(command.blobId())); + } + + private ApplyResult storeObject(TreeMap bucketObjects, ObjectMetadata object, String nodeId, + List alreadyReleased) + { + List released = new ArrayList<>(alreadyReleased); + ObjectMetadata previous = bucketObjects.put(object.key(), object); + holders.put(object.blob().id(), new TreeSet<>(Set.of(nodeId))); + if (previous != null) + { + released.add(release(previous.blob().id())); + } + return new ApplyResult(null, List.of(object.blob()), List.copyOf(released)); + } + + private List removeUpload(String uploadId) + { + UploadMetadata upload = uploads.remove(uploadId); + return upload.parts().values().stream() + .map(part -> release(part.blob().id())) + .toList(); + } + + private String release(String blobId) + { + holders.remove(blobId); + return blobId; + } + + private void clear() + { + nodes.clear(); + buckets.clear(); + objects.clear(); + uploads.clear(); + holders.clear(); + } + + private void writeNodes(MetadataWriter writer) throws IOException + { + writer.writeInt(nodes.size()); + for (Map.Entry node : nodes.entrySet()) + { + writer.writeString(node.getKey()); + writer.writeString(node.getValue()); + } + } + + private void readNodes(MetadataReader reader) throws IOException + { + int count = reader.readInt(); + for (int i = 0; i < count; i++) + { + nodes.put(reader.readString(), reader.readString()); + } + } + + private void writeBuckets(MetadataWriter writer) throws IOException + { + writer.writeInt(buckets.size()); + for (BucketMetadata bucket : buckets.values()) + { + writer.writeString(bucket.name()); + writer.writeInstant(bucket.creationDate()); + writer.writeTags(bucket.tags()); + TreeMap bucketObjects = objects.get(bucket.name()); + writer.writeInt(bucketObjects.size()); + for (ObjectMetadata object : bucketObjects.values()) + { + writer.writeString(object.key()); + writer.writeString(object.etag()); + writer.writeString(object.contentType()); + writer.writeInstant(object.lastModified()); + writer.writeTags(object.tags()); + writer.writeBlob(object.blob()); + } + } + } + + private void readBuckets(MetadataReader reader) throws IOException + { + int bucketCount = reader.readInt(); + for (int i = 0; i < bucketCount; i++) + { + BucketMetadata bucket = new BucketMetadata(reader.readString(), reader.readInstant(), reader.readTags()); + TreeMap bucketObjects = new TreeMap<>(); + int objectCount = reader.readInt(); + for (int j = 0; j < objectCount; j++) + { + ObjectMetadata object = new ObjectMetadata(reader.readString(), reader.readString(), reader.readString(), + reader.readInstant(), reader.readTags(), reader.readBlob()); + bucketObjects.put(object.key(), object); + } + buckets.put(bucket.name(), bucket); + objects.put(bucket.name(), bucketObjects); + } + } + + private void writeUploads(MetadataWriter writer) throws IOException + { + writer.writeInt(uploads.size()); + for (UploadMetadata upload : uploads.values()) + { + writer.writeString(upload.uploadId()); + writer.writeString(upload.bucket()); + writer.writeString(upload.key()); + writer.writeString(upload.contentType()); + writer.writeInstant(upload.initiated()); + writer.writeTags(upload.tags()); + writer.writeInt(upload.parts().size()); + for (PartMetadata part : upload.parts().values()) + { + writer.writeInt(part.partNumber()); + writer.writeString(part.etag()); + writer.writeBlob(part.blob()); + } + } + } + + private void readUploads(MetadataReader reader) throws IOException + { + int uploadCount = reader.readInt(); + for (int i = 0; i < uploadCount; i++) + { + String uploadId = reader.readString(); + String bucket = reader.readString(); + String key = reader.readString(); + String contentType = reader.readString(); + Instant initiated = reader.readInstant(); + List tags = reader.readTags(); + Map parts = new TreeMap<>(); + int partCount = reader.readInt(); + for (int j = 0; j < partCount; j++) + { + PartMetadata part = new PartMetadata(reader.readInt(), reader.readString(), reader.readBlob()); + parts.put(part.partNumber(), part); + } + uploads.put(uploadId, UploadMetadata.of(uploadId, bucket, key, contentType, initiated, tags, parts)); + } + } + + private void writeHolders(MetadataWriter writer) throws IOException + { + writer.writeInt(holders.size()); + for (Map.Entry> blobHolders : holders.entrySet()) + { + writer.writeString(blobHolders.getKey()); + writer.writeStrings(blobHolders.getValue()); + } + } + + private void readHolders(MetadataReader reader) throws IOException + { + int count = reader.readInt(); + for (int i = 0; i < count; i++) + { + holders.put(reader.readString(), new TreeSet<>(reader.readStrings())); + } + } + + private T read(Supplier query) + { + lock.readLock().lock(); + try + { + return query.get(); + } + finally + { + lock.readLock().unlock(); + } + } + + private T write(Supplier update) + { + lock.writeLock().lock(); + try + { + return update.get(); + } + finally + { + lock.writeLock().unlock(); + } + } + + private void signal() + { + synchronized (progress) + { + progress.notifyAll(); + } + } + + private boolean await(BooleanSupplier condition, Duration timeout) + { + long deadline = System.nanoTime() + timeout.toNanos(); + synchronized (progress) + { + try + { + while (!condition.getAsBoolean()) + { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) + { + return false; + } + TimeUnit.NANOSECONDS.timedWait(progress, remaining); + } + return true; + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + return false; + } + } + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/metadata/MetadataWriter.java b/src/main/java/dev/dinauer/r9t/core/metadata/MetadataWriter.java new file mode 100644 index 0000000..383b6a9 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/metadata/MetadataWriter.java @@ -0,0 +1,90 @@ +package dev.dinauer.r9t.core.metadata; + +import dev.dinauer.r9t.core.blob.Blob; +import dev.dinauer.r9t.core.s3.Tag; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Collection; +import java.util.List; + +public class MetadataWriter +{ + private final DataOutputStream output; + + public MetadataWriter(OutputStream output) + { + this.output = new DataOutputStream(output); + } + + public void writeByte(int value) throws IOException + { + output.writeByte(value); + } + + public void writeInt(int value) throws IOException + { + output.writeInt(value); + } + + public void writeLong(long value) throws IOException + { + output.writeLong(value); + } + + public void writeString(String value) throws IOException + { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + output.writeInt(bytes.length); + output.write(bytes); + } + + public void writeNullableString(String value) throws IOException + { + output.writeBoolean(value != null); + if (value != null) + { + writeString(value); + } + } + + public void writeStrings(Collection values) throws IOException + { + output.writeInt(values.size()); + for (String value : values) + { + writeString(value); + } + } + + public void writeInstant(Instant value) throws IOException + { + output.writeLong(value.getEpochSecond()); + output.writeInt(value.getNano()); + } + + public void writeTags(List tags) throws IOException + { + output.writeInt(tags.size()); + for (Tag tag : tags) + { + writeString(tag.key()); + writeString(tag.value()); + } + } + + public void writeBlob(Blob blob) throws IOException + { + writeString(blob.id()); + output.writeLong(blob.size()); + writeString(blob.sha256()); + } + + public void flush() throws IOException + { + output.flush(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/metadata/Proposal.java b/src/main/java/dev/dinauer/r9t/core/metadata/Proposal.java new file mode 100644 index 0000000..71df8ee --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/metadata/Proposal.java @@ -0,0 +1,7 @@ +package dev.dinauer.r9t.core.metadata; + +import java.time.Instant; + +public record Proposal(Instant timestamp, String principal, Command command) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartPart.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartPart.java deleted file mode 100644 index f5e3178..0000000 --- a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartPart.java +++ /dev/null @@ -1,5 +0,0 @@ -package dev.dinauer.r9t.core.multipart; - -public record MultipartPart(int partNumber, String etag, long size) -{ -} diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadMapper.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadMapper.java new file mode 100644 index 0000000..9640cc9 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadMapper.java @@ -0,0 +1,20 @@ +package dev.dinauer.r9t.core.multipart; + +import jakarta.enterprise.context.ApplicationScoped; + +import java.util.List; + +@ApplicationScoped +public class MultipartUploadMapper +{ + public MultipartUpload map(UploadMetadata input) + { + return new MultipartUpload(input.uploadId(), input.bucket(), input.key(), input.contentType(), input.initiated(), + input.tags()); + } + + public List map(List input) + { + return input.stream().map(this::map).toList(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java index aaffc11..33c558d 100644 --- a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadRepo.java @@ -1,157 +1,49 @@ package dev.dinauer.r9t.core.multipart; -import dev.dinauer.r9t.core.bucket.BucketRepo; -import dev.dinauer.r9t.core.s3.DigestCopy; -import dev.dinauer.r9t.core.s3.FileAttributes; -import dev.dinauer.r9t.core.s3.S3ErrorCode; -import dev.dinauer.r9t.core.s3.S3Exception; -import dev.dinauer.r9t.core.s3.Tag; -import dev.dinauer.r9t.core.s3.TagCodec; +import dev.dinauer.r9t.core.blob.Blob; +import dev.dinauer.r9t.core.metadata.Command; +import dev.dinauer.r9t.core.metadata.MetadataStore; +import dev.dinauer.r9t.core.raft.RaftService; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.Instant; -import java.util.Comparator; -import java.util.List; -import java.util.UUID; -import java.util.stream.Stream; +import java.util.Optional; @ApplicationScoped public class MultipartUploadRepo { - private static final String BUCKET_ATTRIBUTE = "user.bucket"; - private static final String KEY_ATTRIBUTE = "user.key"; - private static final String INITIATED_ATTRIBUTE = "user.initiated"; + @Inject + RaftService raftService; @Inject - BucketRepo bucketRepo; + MetadataStore metadataStore; - public Path resolve(String uploadId) + public Optional find(String uploadId) { - Path root = stagingRoot(); - Path target = root.resolve(uploadId).normalize(); - if (!target.startsWith(root)) - { - throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, uploadId); - } - return target; + return raftService.read(() -> metadataStore.upload(uploadId)); } - public boolean exists(String uploadId) + public void create(UploadMetadata upload) { - return Files.isDirectory(resolve(uploadId)); + raftService.submit(new Command.InitiateUpload(upload.uploadId(), upload.bucket(), upload.key(), upload.contentType(), + upload.tags(), upload.initiated()), upload.uploadId()); } - public MultipartUpload create(String bucket, String key, String contentType, List tags) + public void putPart(String uploadId, PartMetadata part) { - String uploadId = UUID.randomUUID().toString().replace("-", ""); - Path dir = resolve(uploadId); - Instant initiated = Instant.now(); - try - { - Files.createDirectories(dir); - FileAttributes.write(dir, BUCKET_ATTRIBUTE, bucket); - FileAttributes.write(dir, KEY_ATTRIBUTE, key); - FileAttributes.write(dir, FileAttributes.CONTENT_TYPE, contentType); - FileAttributes.write(dir, INITIATED_ATTRIBUTE, initiated.toString()); - FileAttributes.write(dir, FileAttributes.TAGS, TagCodec.toQueryString(tags)); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); - } - return new MultipartUpload(uploadId, bucket, key, contentType, initiated, tags); + raftService.submit(new Command.PutPart(uploadId, part.partNumber(), part.etag(), part.blob(), raftService.selfId()), + uploadId); } - public MultipartUpload metadata(String uploadId) + public void complete(UploadMetadata upload, String etag, Instant lastModified, Blob blob) { - Path dir = resolve(uploadId); - try - { - String bucket = FileAttributes.read(dir, BUCKET_ATTRIBUTE); - String key = FileAttributes.read(dir, KEY_ATTRIBUTE); - String contentType = FileAttributes.read(dir, FileAttributes.CONTENT_TYPE); - Instant initiated = Instant.parse(FileAttributes.read(dir, INITIATED_ATTRIBUTE)); - List tags = TagCodec.parseQueryString(FileAttributes.read(dir, FileAttributes.TAGS)); - return new MultipartUpload(uploadId, bucket, key, contentType, initiated, tags); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); - } + raftService.submit(new Command.CompleteUpload(upload.uploadId(), upload.bucket(), upload.key(), etag, lastModified, blob, + raftService.selfId()), upload.uploadId()); } - public String storePart(String uploadId, int partNumber, InputStream data) + public void abort(String uploadId) { - Path partFile = partFile(uploadId, partNumber); - try - { - String etag = DigestCopy.copyAndHashMd5(data, partFile); - FileAttributes.write(partFile, FileAttributes.ETAG, etag); - return etag; - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); - } - } - - public boolean partExists(String uploadId, int partNumber) - { - return Files.isRegularFile(partFile(uploadId, partNumber)); - } - - public MultipartPart part(String uploadId, int partNumber) - { - Path partFile = partFile(uploadId, partNumber); - try - { - String etag = FileAttributes.read(partFile, FileAttributes.ETAG); - long size = Files.size(partFile); - return new MultipartPart(partNumber, etag, size); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); - } - } - - public Path partFile(String uploadId, int partNumber) - { - return resolve(uploadId).resolve(String.format("%05d", partNumber)); - } - - public void delete(String uploadId) - { - Path dir = resolve(uploadId); - try (Stream entries = Files.walk(dir)) - { - for (Path path : entries.sorted(Comparator.reverseOrder()).toList()) - { - Files.deleteIfExists(path); - } - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); - } - } - - private Path stagingRoot() - { - Path root = bucketRepo.root().resolve(".multipart"); - try - { - Files.createDirectories(root); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, root.toString(), e); - } - return root; + raftService.submit(new Command.AbortUpload(uploadId), uploadId); } } diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java index 471a8f2..3371fef 100644 --- a/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java +++ b/src/main/java/dev/dinauer/r9t/core/multipart/MultipartUploadService.java @@ -1,10 +1,9 @@ package dev.dinauer.r9t.core.multipart; -import dev.dinauer.r9t.core.audit.AuditAction; -import dev.dinauer.r9t.core.audit.AuditLogService; -import dev.dinauer.r9t.core.audit.AuditResourceType; +import dev.dinauer.r9t.core.blob.BlobService; +import dev.dinauer.r9t.core.blob.StoredBlob; import dev.dinauer.r9t.core.bucket.BucketService; -import dev.dinauer.r9t.core.object.ObjectRepo; +import dev.dinauer.r9t.core.object.ObjectService; import dev.dinauer.r9t.core.object.S3Object; import dev.dinauer.r9t.core.s3.DigestCopy; import dev.dinauer.r9t.core.s3.S3ErrorCode; @@ -14,18 +13,14 @@ import dev.dinauer.r9t.core.s3.TagValidator; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import java.io.IOException; import java.io.InputStream; -import java.io.SequenceInputStream; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.security.MessageDigest; +import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.HexFormat; import java.util.List; -import java.util.Vector; +import java.util.Map; +import java.util.UUID; @ApplicationScoped public class MultipartUploadService @@ -34,24 +29,26 @@ public class MultipartUploadService MultipartUploadRepo multipartUploadRepo; @Inject - ObjectRepo objectRepo; + MultipartUploadMapper multipartUploadMapper; + + @Inject + ObjectService objectService; @Inject BucketService bucketService; @Inject - AuditLogService auditLogService; + BlobService blobService; public MultipartUpload initiate(String bucket, String key, String contentType, List tags) { bucketService.require(bucket); - String resolvedContentType = contentType; - if (resolvedContentType == null) - { - resolvedContentType = "application/octet-stream"; - } TagValidator.validate(tags); - return multipartUploadRepo.create(bucket, key, resolvedContentType, tags); + String uploadId = UUID.randomUUID().toString().replace("-", ""); + UploadMetadata upload = UploadMetadata.of(uploadId, bucket, key, objectService.resolveContentType(contentType), + Instant.now(), tags, Map.of()); + multipartUploadRepo.create(upload); + return multipartUploadMapper.map(upload); } public String uploadPart(String bucket, String key, String uploadId, Integer partNumber, InputStream data) @@ -61,49 +58,38 @@ public class MultipartUploadService { throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, "partNumber"); } - return multipartUploadRepo.storePart(uploadId, partNumber, data); + StoredBlob stored = blobService.store(data); + PartMetadata part = new PartMetadata(partNumber, stored.md5(), stored.blob()); + blobService.commit(stored.blob(), () -> multipartUploadRepo.putPart(uploadId, part)); + return stored.md5(); } public S3Object complete(String bucket, String key, String uploadId, List parts) { - MultipartUpload upload = require(bucket, key, uploadId); + UploadMetadata upload = require(bucket, key, uploadId); if (parts.isEmpty()) { throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, uploadId); } - List resolvedParts = resolveParts(uploadId, parts); + List resolvedParts = resolveParts(upload, parts); String etag = combineEtags(resolvedParts); - try (InputStream assembled = assemble(uploadId, resolvedParts)) - { - objectRepo.storeWithEtag(bucket, key, upload.contentType(), assembled, etag); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, uploadId, e); - } - if (!upload.tags().isEmpty()) - { - objectRepo.putTags(bucket, key, upload.tags()); - } - multipartUploadRepo.delete(uploadId); - auditLogService.record(AuditAction.CREATE, AuditResourceType.OBJECT, bucket, key); - return objectRepo.metadata(bucket, key); + StoredBlob stored = blobService.assemble(resolvedParts.stream().map(PartMetadata::blob).toList()); + blobService.commit(stored.blob(), () -> multipartUploadRepo.complete(upload, etag, Instant.now(), stored.blob())); + blobService.awaitReplicas(stored.blob(), bucket + "/" + key); + return objectService.metadata(bucket, key); } public void abort(String bucket, String key, String uploadId) { require(bucket, key, uploadId); - multipartUploadRepo.delete(uploadId); + multipartUploadRepo.abort(uploadId); } - private MultipartUpload require(String bucket, String key, String uploadId) + private UploadMetadata require(String bucket, String key, String uploadId) { bucketService.require(bucket); - if (!multipartUploadRepo.exists(uploadId)) - { - throw new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD, uploadId); - } - MultipartUpload upload = multipartUploadRepo.metadata(uploadId); + UploadMetadata upload = multipartUploadRepo.find(uploadId) + .orElseThrow(() -> new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD, uploadId)); if (!upload.bucket().equals(bucket) || !upload.key().equals(key)) { throw new S3Exception(S3ErrorCode.NO_SUCH_UPLOAD, uploadId); @@ -111,24 +97,20 @@ public class MultipartUploadService return upload; } - private List resolveParts(String uploadId, List parts) + private List resolveParts(UploadMetadata upload, List parts) { - List resolved = new ArrayList<>(); + List resolved = new ArrayList<>(); int previousPartNumber = 0; for (CompletedPart requested : parts) { if (requested.partNumber() <= previousPartNumber) { - throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId); + throw new S3Exception(S3ErrorCode.INVALID_PART, upload.uploadId()); } - if (!multipartUploadRepo.partExists(uploadId, requested.partNumber())) + PartMetadata stored = upload.parts().get(requested.partNumber()); + if (stored == null || !stored.etag().equalsIgnoreCase(requested.etag())) { - throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId); - } - MultipartPart stored = multipartUploadRepo.part(uploadId, requested.partNumber()); - if (!stored.etag().equalsIgnoreCase(requested.etag())) - { - throw new S3Exception(S3ErrorCode.INVALID_PART, uploadId); + throw new S3Exception(S3ErrorCode.INVALID_PART, upload.uploadId()); } resolved.add(stored); previousPartNumber = requested.partNumber(); @@ -136,31 +118,13 @@ public class MultipartUploadService return resolved; } - private String combineEtags(List parts) + private String combineEtags(List parts) { MessageDigest digest = DigestCopy.newMd5(); - for (MultipartPart part : parts) + for (PartMetadata part : parts) { digest.update(HexFormat.of().parseHex(part.etag())); } return HexFormat.of().formatHex(digest.digest()) + "-" + parts.size(); } - - private InputStream assemble(String uploadId, List parts) - { - Vector streams = new Vector<>(); - for (MultipartPart part : parts) - { - Path partFile = multipartUploadRepo.partFile(uploadId, part.partNumber()); - try - { - streams.add(Files.newInputStream(partFile)); - } - catch (IOException e) - { - throw new UncheckedIOException(e); - } - } - return new SequenceInputStream(Collections.enumeration(streams)); - } } diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/PartMetadata.java b/src/main/java/dev/dinauer/r9t/core/multipart/PartMetadata.java new file mode 100644 index 0000000..1db3296 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/multipart/PartMetadata.java @@ -0,0 +1,7 @@ +package dev.dinauer.r9t.core.multipart; + +import dev.dinauer.r9t.core.blob.Blob; + +public record PartMetadata(int partNumber, String etag, Blob blob) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/multipart/UploadMetadata.java b/src/main/java/dev/dinauer/r9t/core/multipart/UploadMetadata.java new file mode 100644 index 0000000..d1e2aa0 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/multipart/UploadMetadata.java @@ -0,0 +1,26 @@ +package dev.dinauer.r9t.core.multipart; + +import dev.dinauer.r9t.core.s3.Tag; + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +public record UploadMetadata(String uploadId, String bucket, String key, String contentType, Instant initiated, + List tags, SortedMap parts) +{ + public UploadMetadata withPart(PartMetadata part) + { + TreeMap newParts = new TreeMap<>(parts); + newParts.put(part.partNumber(), part); + return new UploadMetadata(uploadId, bucket, key, contentType, initiated, tags, newParts); + } + + public static UploadMetadata of(String uploadId, String bucket, String key, String contentType, Instant initiated, + List tags, Map parts) + { + return new UploadMetadata(uploadId, bucket, key, contentType, initiated, List.copyOf(tags), new TreeMap<>(parts)); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectContent.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectContent.java new file mode 100644 index 0000000..d97d2c7 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectContent.java @@ -0,0 +1,7 @@ +package dev.dinauer.r9t.core.object; + +import java.io.InputStream; + +public record ObjectContent(S3Object metadata, InputStream data) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectMetadata.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectMetadata.java new file mode 100644 index 0000000..46ed61b --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectMetadata.java @@ -0,0 +1,15 @@ +package dev.dinauer.r9t.core.object; + +import dev.dinauer.r9t.core.blob.Blob; +import dev.dinauer.r9t.core.s3.Tag; + +import java.time.Instant; +import java.util.List; + +public record ObjectMetadata(String key, String etag, String contentType, Instant lastModified, List tags, Blob blob) +{ + public ObjectMetadata withTags(List newTags) + { + return new ObjectMetadata(key, etag, contentType, lastModified, List.copyOf(newTags), blob); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java index 91d4944..17f2a45 100644 --- a/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectRepo.java @@ -1,264 +1,47 @@ package dev.dinauer.r9t.core.object; -import dev.dinauer.r9t.core.bucket.BucketRepo; -import dev.dinauer.r9t.core.s3.DigestCopy; -import dev.dinauer.r9t.core.s3.FileAttributes; -import dev.dinauer.r9t.core.s3.S3ErrorCode; -import dev.dinauer.r9t.core.s3.S3Exception; +import dev.dinauer.r9t.core.metadata.Command; +import dev.dinauer.r9t.core.metadata.MetadataStore; +import dev.dinauer.r9t.core.raft.RaftService; import dev.dinauer.r9t.core.s3.Tag; -import dev.dinauer.r9t.core.s3.TagCodec; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; -import java.util.stream.Stream; +import java.util.Optional; @ApplicationScoped public class ObjectRepo { @Inject - BucketRepo bucketRepo; + RaftService raftService; - public Path resolve(String bucket, String key) + @Inject + MetadataStore metadataStore; + + public Optional find(String bucket, String key) { - Path bucketPath = bucketRepo.resolve(bucket); - Path target = bucketPath.resolve(key).normalize(); - if (!target.startsWith(bucketPath)) - { - throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, key); - } - return target; + return raftService.read(() -> metadataStore.object(bucket, key)); } - public boolean exists(String bucket, String key) + public List list(String bucket, String prefix) { - return Files.isRegularFile(resolve(bucket, key)); + return raftService.read(() -> metadataStore.objects(bucket, prefix)); } - public String put(String bucket, String key, String contentType, InputStream data) + public void put(String bucket, ObjectMetadata object) { - Path target = resolve(bucket, key); - try - { - Files.createDirectories(target.getParent()); - String etag = DigestCopy.copyAndHashMd5(data, target); - FileAttributes.write(target, FileAttributes.ETAG, etag); - FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType); - clearTags(target); - return etag; - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); - } - } - - public void storeWithEtag(String bucket, String key, String contentType, InputStream data, String etag) - { - Path target = resolve(bucket, key); - try - { - Files.createDirectories(target.getParent()); - Files.copy(data, target, StandardCopyOption.REPLACE_EXISTING); - FileAttributes.write(target, FileAttributes.ETAG, etag); - FileAttributes.write(target, FileAttributes.CONTENT_TYPE, contentType); - clearTags(target); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); - } - } - - public List getTags(String bucket, String key) - { - Path target = resolve(bucket, key); - try - { - if (FileAttributes.exists(target, FileAttributes.TAGS)) - { - return TagCodec.parseQueryString(FileAttributes.read(target, FileAttributes.TAGS)); - } - return List.of(); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); - } - } - - public void putTags(String bucket, String key, List tags) - { - Path target = resolve(bucket, key); - try - { - FileAttributes.write(target, FileAttributes.TAGS, TagCodec.toQueryString(tags)); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); - } - } - - public void deleteTags(String bucket, String key) - { - Path target = resolve(bucket, key); - try - { - clearTags(target); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); - } - } - - private void clearTags(Path target) throws IOException - { - if (FileAttributes.exists(target, FileAttributes.TAGS)) - { - FileAttributes.delete(target, FileAttributes.TAGS); - } - } - - public S3Object metadata(String bucket, String key) - { - Path target = resolve(bucket, key); - try - { - BasicFileAttributes attributes = Files.readAttributes(target, BasicFileAttributes.class); - String etag = FileAttributes.read(target, FileAttributes.ETAG); - String contentType = FileAttributes.read(target, FileAttributes.CONTENT_TYPE); - return new S3Object(key, attributes.size(), attributes.lastModifiedTime().toInstant(), etag, contentType); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); - } - } - - public Path file(String bucket, String key) - { - return resolve(bucket, key); + raftService.submit(new Command.PutObject(bucket, object.key(), object.etag(), object.contentType(), object.tags(), + object.lastModified(), object.blob(), raftService.selfId()), bucket + "/" + object.key()); } public void delete(String bucket, String key) { - try - { - Files.deleteIfExists(resolve(bucket, key)); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket + "/" + key, e); - } + raftService.submit(new Command.DeleteObject(bucket, key), bucket + "/" + key); } - public List searchByTags(String bucket, Map tagFilters) + public void putTags(String bucket, String key, List tags) { - Path bucketPath = bucketRepo.resolve(bucket); - try (Stream entries = Files.walk(bucketPath)) - { - return entries - .filter(Files::isRegularFile) - .map(path -> bucketPath.relativize(path).toString()) - .filter(key -> matchesAllTags(bucket, key, tagFilters)) - .sorted() - .map(key -> metadata(bucket, key)) - .toList(); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); - } - } - - private boolean matchesAllTags(String bucket, String key, Map tagFilters) - { - List tags = getTags(bucket, key); - for (Map.Entry filter : tagFilters.entrySet()) - { - if (!matchesTag(tags, filter.getKey(), filter.getValue())) - { - return false; - } - } - return true; - } - - private boolean matchesTag(List tags, String key, String value) - { - return tags.stream().anyMatch(tag -> tag.key().equals(key) && tag.value().equals(value)); - } - - public ListObjectsResult list(String bucket, String prefix, String delimiter) - { - Path bucketPath = bucketRepo.resolve(bucket); - try (Stream entries = Files.walk(bucketPath)) - { - List keys = entries - .filter(Files::isRegularFile) - .map(path -> bucketPath.relativize(path).toString()) - .filter(key -> matchesPrefix(key, prefix)) - .sorted() - .toList(); - return group(bucket, keys, normalizedPrefix(prefix), delimiter); - } - catch (IOException e) - { - throw new S3Exception(S3ErrorCode.INTERNAL_ERROR, bucket, e); - } - } - - private boolean matchesPrefix(String key, String prefix) - { - if (prefix == null) - { - return true; - } - return key.startsWith(prefix); - } - - private String normalizedPrefix(String prefix) - { - if (prefix == null) - { - return ""; - } - return prefix; - } - - private ListObjectsResult group(String bucket, List keys, String prefix, String delimiter) - { - if (delimiter == null) - { - List contents = keys.stream().map(key -> metadata(bucket, key)).toList(); - return new ListObjectsResult(contents, List.of()); - } - List contents = new ArrayList<>(); - LinkedHashSet commonPrefixes = new LinkedHashSet<>(); - for (String key : keys) - { - String rest = key.substring(prefix.length()); - int delimiterIndex = rest.indexOf(delimiter); - if (delimiterIndex == -1) - { - contents.add(metadata(bucket, key)); - } - else - { - commonPrefixes.add(prefix + rest.substring(0, delimiterIndex + delimiter.length())); - } - } - return new ListObjectsResult(contents, List.copyOf(commonPrefixes)); + raftService.submit(new Command.PutObjectTags(bucket, key, tags), bucket + "/" + key); } } diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java index c7db1d2..249d6d0 100644 --- a/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectResource.java @@ -119,11 +119,7 @@ public class ObjectResource String partEtag = multipartUploadService.uploadPart(bucket, key, uploadId, partNumber, payload); return Response.ok().header("ETag", "\"" + partEtag + "\"").build(); } - String etag = objectService.put(bucket, key, contentType, payload); - if (tagging != null) - { - objectService.putTags(bucket, key, TagCodec.parseQueryString(tagging)); - } + String etag = objectService.put(bucket, key, contentType, TagCodec.parseQueryString(tagging), payload); return Response.ok().header("ETag", "\"" + etag + "\"").build(); } @@ -156,10 +152,12 @@ public class ObjectResource List tags = objectService.getTags(bucket, key); return Response.ok(objectXmlMapper.mapTagging(tags)).type(MediaType.APPLICATION_XML).build(); } - S3Object metadata = objectService.metadata(bucket, key); - return Response.ok(objectService.file(bucket, key).toFile()) + ObjectContent content = objectService.content(bucket, key); + S3Object metadata = content.metadata(); + return Response.ok(content.data()) .type(metadata.contentType()) .header("ETag", "\"" + metadata.etag() + "\"") + .header("Content-Length", metadata.size()) .lastModified(Date.from(metadata.lastModified())) .build(); } diff --git a/src/main/java/dev/dinauer/r9t/core/object/ObjectService.java b/src/main/java/dev/dinauer/r9t/core/object/ObjectService.java index 2855187..4d53f49 100644 --- a/src/main/java/dev/dinauer/r9t/core/object/ObjectService.java +++ b/src/main/java/dev/dinauer/r9t/core/object/ObjectService.java @@ -1,8 +1,7 @@ package dev.dinauer.r9t.core.object; -import dev.dinauer.r9t.core.audit.AuditAction; -import dev.dinauer.r9t.core.audit.AuditLogService; -import dev.dinauer.r9t.core.audit.AuditResourceType; +import dev.dinauer.r9t.core.blob.BlobService; +import dev.dinauer.r9t.core.blob.StoredBlob; import dev.dinauer.r9t.core.bucket.BucketService; import dev.dinauer.r9t.core.s3.S3ErrorCode; import dev.dinauer.r9t.core.s3.S3Exception; @@ -12,7 +11,9 @@ import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.io.InputStream; -import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -26,12 +27,20 @@ public class ObjectService BucketService bucketService; @Inject - AuditLogService auditLogService; + BlobService blobService; + + @Inject + S3ObjectMapper s3ObjectMapper; public ListObjectsResult list(String bucket, String prefix, String delimiter) { bucketService.require(bucket); - return objectRepo.list(bucket, prefix, delimiter); + List objects = objectRepo.list(bucket, prefix); + if (delimiter == null) + { + return new ListObjectsResult(s3ObjectMapper.map(objects), List.of()); + } + return group(objects, normalizedPrefix(prefix), delimiter); } public List searchByTags(String bucket, Map tagFilters) @@ -41,61 +50,105 @@ public class ObjectService { throw new S3Exception(S3ErrorCode.INVALID_ARGUMENT, "x-tag-*"); } - return objectRepo.searchByTags(bucket, tagFilters); + return s3ObjectMapper.map(objectRepo.list(bucket, null).stream() + .filter(object -> matchesAllTags(object.tags(), tagFilters)) + .toList()); } - public String put(String bucket, String key, String contentType, InputStream data) + public String put(String bucket, String key, String contentType, List tags, InputStream data) { bucketService.require(bucket); - String resolvedContentType = contentType; - if (resolvedContentType == null) - { - resolvedContentType = "application/octet-stream"; - } - String etag = objectRepo.put(bucket, key, resolvedContentType, data); - auditLogService.record(AuditAction.CREATE, AuditResourceType.OBJECT, bucket, key); - return etag; + TagValidator.validate(tags); + StoredBlob stored = blobService.store(data); + ObjectMetadata object = new ObjectMetadata(key, stored.md5(), resolveContentType(contentType), Instant.now(), + List.copyOf(tags), stored.blob()); + blobService.commit(stored.blob(), () -> objectRepo.put(bucket, object)); + blobService.awaitReplicas(stored.blob(), bucket + "/" + key); + return stored.md5(); } public S3Object metadata(String bucket, String key) { - bucketService.require(bucket); - if (!objectRepo.exists(bucket, key)) - { - throw new S3Exception(S3ErrorCode.NO_SUCH_KEY, bucket + "/" + key); - } - return objectRepo.metadata(bucket, key); + return s3ObjectMapper.map(require(bucket, key)); } - public Path file(String bucket, String key) + public ObjectContent content(String bucket, String key) { - metadata(bucket, key); - return objectRepo.file(bucket, key); + ObjectMetadata object = require(bucket, key); + return new ObjectContent(s3ObjectMapper.map(object), blobService.open(object.blob())); } public void delete(String bucket, String key) { bucketService.require(bucket); objectRepo.delete(bucket, key); - auditLogService.record(AuditAction.DELETE, AuditResourceType.OBJECT, bucket, key); } public List getTags(String bucket, String key) { - metadata(bucket, key); - return objectRepo.getTags(bucket, key); + return require(bucket, key).tags(); } public void putTags(String bucket, String key, List tags) { - metadata(bucket, key); + require(bucket, key); TagValidator.validate(tags); objectRepo.putTags(bucket, key, tags); } public void deleteTags(String bucket, String key) { - metadata(bucket, key); - objectRepo.deleteTags(bucket, key); + require(bucket, key); + objectRepo.putTags(bucket, key, List.of()); + } + + public String resolveContentType(String contentType) + { + if (contentType == null) + { + return "application/octet-stream"; + } + return contentType; + } + + private ObjectMetadata require(String bucket, String key) + { + bucketService.require(bucket); + return objectRepo.find(bucket, key).orElseThrow(() -> new S3Exception(S3ErrorCode.NO_SUCH_KEY, bucket + "/" + key)); + } + + private boolean matchesAllTags(List tags, Map tagFilters) + { + return tagFilters.entrySet().stream() + .allMatch(filter -> tags.stream().anyMatch(tag -> tag.key().equals(filter.getKey()) && tag.value().equals(filter.getValue()))); + } + + private String normalizedPrefix(String prefix) + { + if (prefix == null) + { + return ""; + } + return prefix; + } + + private ListObjectsResult group(List objects, String prefix, String delimiter) + { + List contents = new ArrayList<>(); + LinkedHashSet commonPrefixes = new LinkedHashSet<>(); + for (ObjectMetadata object : objects) + { + String rest = object.key().substring(prefix.length()); + int delimiterIndex = rest.indexOf(delimiter); + if (delimiterIndex == -1) + { + contents.add(s3ObjectMapper.map(object)); + } + else + { + commonPrefixes.add(prefix + rest.substring(0, delimiterIndex + delimiter.length())); + } + } + return new ListObjectsResult(contents, List.copyOf(commonPrefixes)); } } diff --git a/src/main/java/dev/dinauer/r9t/core/object/S3ObjectMapper.java b/src/main/java/dev/dinauer/r9t/core/object/S3ObjectMapper.java new file mode 100644 index 0000000..3bc016c --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/object/S3ObjectMapper.java @@ -0,0 +1,19 @@ +package dev.dinauer.r9t.core.object; + +import jakarta.enterprise.context.ApplicationScoped; + +import java.util.List; + +@ApplicationScoped +public class S3ObjectMapper +{ + public S3Object map(ObjectMetadata input) + { + return new S3Object(input.key(), input.blob().size(), input.lastModified(), input.etag(), input.contentType()); + } + + public List map(List input) + { + return input.stream().map(this::map).toList(); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/raft/MetadataStateMachine.java b/src/main/java/dev/dinauer/r9t/core/raft/MetadataStateMachine.java new file mode 100644 index 0000000..f0703f8 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/raft/MetadataStateMachine.java @@ -0,0 +1,151 @@ +package dev.dinauer.r9t.core.raft; + +import dev.dinauer.r9t.core.audit.AuditAction; +import dev.dinauer.r9t.core.audit.AuditLogEntry; +import dev.dinauer.r9t.core.audit.AuditLogService; +import dev.dinauer.r9t.core.audit.AuditResourceType; +import dev.dinauer.r9t.core.blob.BlobReplicator; +import dev.dinauer.r9t.core.blob.BlobRepo; +import dev.dinauer.r9t.core.metadata.ApplyResult; +import dev.dinauer.r9t.core.metadata.Command; +import dev.dinauer.r9t.core.metadata.CommandCodec; +import dev.dinauer.r9t.core.metadata.MetadataStore; +import dev.dinauer.r9t.core.metadata.Proposal; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; +import org.apache.ratis.protocol.Message; +import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.server.RaftServer; +import org.apache.ratis.server.protocol.TermIndex; +import org.apache.ratis.server.storage.FileInfo; +import org.apache.ratis.server.storage.RaftStorage; +import org.apache.ratis.statemachine.StateMachineStorage; +import org.apache.ratis.statemachine.TransactionContext; +import org.apache.ratis.statemachine.impl.BaseStateMachine; +import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage; +import org.apache.ratis.statemachine.impl.SingleFileSnapshotInfo; +import org.apache.ratis.util.MD5FileUtil; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.concurrent.CompletableFuture; + +@Singleton +public class MetadataStateMachine extends BaseStateMachine +{ + private final SimpleStateMachineStorage storage = new SimpleStateMachineStorage(); + + @Inject + MetadataStore metadataStore; + + @Inject + CommandCodec commandCodec; + + @Inject + BlobRepo blobRepo; + + @Inject + BlobReplicator blobReplicator; + + @Inject + AuditLogService auditLogService; + + @Override + public void initialize(RaftServer server, RaftGroupId groupId, RaftStorage raftStorage) throws IOException + { + super.initialize(server, groupId, raftStorage); + storage.init(raftStorage); + load(); + } + + @Override + public void reinitialize() throws IOException + { + load(); + } + + @Override + public StateMachineStorage getStateMachineStorage() + { + return storage; + } + + @Override + public synchronized CompletableFuture applyTransaction(TransactionContext transaction) + { + LogEntryProto entry = transaction.getLogEntry(); + Proposal proposal = commandCodec.decode(entry.getStateMachineLogEntry().getLogData().toByteArray()); + ApplyResult result = metadataStore.apply(proposal.command(), entry.getIndex()); + updateLastAppliedTermIndex(entry.getTerm(), entry.getIndex()); + result.released().forEach(blobRepo::delete); + result.added().forEach(blobReplicator::schedule); + if (result.success()) + { + audit(proposal, entry.getIndex()); + } + return CompletableFuture.completedFuture(Message.valueOf(result.code())); + } + + @Override + public synchronized long takeSnapshot() throws IOException + { + TermIndex last = getLastAppliedTermIndex(); + File target = storage.getSnapshotFile(last.getTerm(), last.getIndex()); + Path tmp = target.toPath().resolveSibling(target.getName() + ".tmp"); + try (OutputStream output = new BufferedOutputStream(Files.newOutputStream(tmp))) + { + metadataStore.writeSnapshot(output); + } + Files.move(tmp, target.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + FileInfo info = new FileInfo(target.toPath(), MD5FileUtil.computeAndSaveMd5ForFile(target)); + storage.updateLatestSnapshot(new SingleFileSnapshotInfo(info, last)); + return last.getIndex(); + } + + @Override + public CompletableFuture query(Message request) + { + return CompletableFuture.completedFuture(Message.EMPTY); + } + + private void load() throws IOException + { + SingleFileSnapshotInfo snapshot = storage.loadLatestSnapshot(); + if (snapshot != null) + { + try (InputStream input = new BufferedInputStream(Files.newInputStream(snapshot.getFile().getPath()))) + { + metadataStore.readSnapshot(input, snapshot.getIndex()); + } + setLastAppliedTermIndex(snapshot.getTermIndex()); + } + } + + private void audit(Proposal proposal, long index) + { + switch (proposal.command()) + { + case Command.CreateBucket c -> record(proposal, index, AuditAction.CREATE, AuditResourceType.BUCKET, c.name(), null); + case Command.DeleteBucket c -> record(proposal, index, AuditAction.DELETE, AuditResourceType.BUCKET, c.name(), null); + case Command.PutObject c -> record(proposal, index, AuditAction.CREATE, AuditResourceType.OBJECT, c.bucket(), c.key()); + case Command.CompleteUpload c -> record(proposal, index, AuditAction.CREATE, AuditResourceType.OBJECT, c.bucket(), c.key()); + case Command.DeleteObject c -> record(proposal, index, AuditAction.DELETE, AuditResourceType.OBJECT, c.bucket(), c.key()); + default -> + { + } + } + } + + private void record(Proposal proposal, long index, AuditAction action, AuditResourceType type, String bucket, String key) + { + auditLogService.record(new AuditLogEntry(proposal.timestamp(), action, type, bucket, key, proposal.principal()), index); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/raft/RaftNode.java b/src/main/java/dev/dinauer/r9t/core/raft/RaftNode.java new file mode 100644 index 0000000..cb7888b --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/raft/RaftNode.java @@ -0,0 +1,21 @@ +package dev.dinauer.r9t.core.raft; + +public record RaftNode(String id, String host, int port) +{ + public static RaftNode parse(String entry) + { + int separator = entry.lastIndexOf(':'); + if (separator <= 0 || separator == entry.length() - 1) + { + throw new IllegalArgumentException("Invalid raft node '" + entry + "', expected host:port"); + } + String host = entry.substring(0, separator); + int port = Integer.parseInt(entry.substring(separator + 1)); + return new RaftNode(host + "_" + port, host, port); + } + + public boolean matches(String entry) + { + return entry.equalsIgnoreCase(host) || entry.equalsIgnoreCase(host + ":" + port); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/raft/RaftService.java b/src/main/java/dev/dinauer/r9t/core/raft/RaftService.java new file mode 100644 index 0000000..fb69f2a --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/raft/RaftService.java @@ -0,0 +1,392 @@ +package dev.dinauer.r9t.core.raft; + +import dev.dinauer.r9t.core.audit.AuditLogService; +import dev.dinauer.r9t.core.metadata.Command; +import dev.dinauer.r9t.core.metadata.CommandCodec; +import dev.dinauer.r9t.core.metadata.MetadataStore; +import dev.dinauer.r9t.core.metadata.Proposal; +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.S3Exception; +import io.quarkus.runtime.ShutdownEvent; +import io.quarkus.runtime.StartupEvent; +import jakarta.annotation.Priority; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.event.Observes; +import jakarta.inject.Inject; +import jakarta.interceptor.Interceptor; +import org.apache.ratis.RaftConfigKeys; +import org.apache.ratis.client.RaftClient; +import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.grpc.GrpcConfigKeys; +import org.apache.ratis.protocol.Message; +import org.apache.ratis.protocol.RaftClientReply; +import org.apache.ratis.protocol.RaftGroup; +import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftPeer; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.retry.RetryPolicies; +import org.apache.ratis.rpc.SupportedRpcType; +import org.apache.ratis.server.DivisionInfo; +import org.apache.ratis.server.RaftServer; +import org.apache.ratis.server.RaftServerConfigKeys; +import org.apache.ratis.server.storage.RaftStorage; +import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; +import org.apache.ratis.util.TimeDuration; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +@ApplicationScoped +public class RaftService +{ + private static final Logger LOG = Logger.getLogger(RaftService.class); + private static final RaftGroupId GROUP_ID = RaftGroupId.valueOf(UUID.nameUUIDFromBytes("r9t".getBytes(StandardCharsets.UTF_8))); + private static final String SINGLE_NODE_ID = "r9t"; + private static final String SINGLE_NODE_HOST = "127.0.0.1"; + private static final int CLIENT_RETRIES = 20; + private static final long SNAPSHOT_THRESHOLD = 1000; + private static final Duration READY_POLL_INTERVAL = Duration.ofMillis(200); + + @ConfigProperty(name = "dev.dinauer.r9t.raft.nodes") + Optional> nodes; + + @ConfigProperty(name = "dev.dinauer.r9t.raft.self") + Optional self; + + @ConfigProperty(name = "dev.dinauer.r9t.raft.port", defaultValue = "9870") + int port; + + @ConfigProperty(name = "dev.dinauer.r9t.raft.advertised-url") + Optional advertisedUrl; + + @ConfigProperty(name = "quarkus.http.port", defaultValue = "8080") + int httpPort; + + @ConfigProperty(name = "dev.dinauer.r9t.raft.timeout", defaultValue = "PT15S") + Duration timeout; + + @ConfigProperty(name = "dev.dinauer.r9t.workdir", defaultValue = "/var/lib/r9t") + String workdir; + + @Inject + MetadataStateMachine stateMachine; + + @Inject + MetadataStore metadataStore; + + @Inject + CommandCodec commandCodec; + + @Inject + AuditLogService auditLogService; + + private RaftNode selfNode; + private RaftGroup group; + private RaftServer server; + private RaftClient client; + + void start(@Observes @Priority(Interceptor.Priority.PLATFORM_BEFORE) StartupEvent event) throws IOException + { + List cluster = cluster(); + selfNode = resolveSelf(cluster); + group = RaftGroup.valueOf(GROUP_ID, cluster.stream().map(this::peer).toList()); + RaftProperties properties = properties(); + boolean existing = Files.isDirectory(storageDir().resolve(GROUP_ID.getUuid().toString())); + if (!existing) + { + auditLogService.resetIndex(); + } + server = RaftServer.newBuilder() + .setServerId(RaftPeerId.valueOf(selfNode.id())) + .setGroup(group) + .setStateMachine(stateMachine) + .setProperties(properties) + .setOption(startupOption(existing)) + .build(); + server.start(); + verifyMembership(); + client = RaftClient.newBuilder() + .setProperties(properties) + .setRaftGroup(group) + .setRetryPolicy(RetryPolicies.retryUpToMaximumCountWithFixedSleep(CLIENT_RETRIES, + TimeDuration.valueOf(500, TimeUnit.MILLISECONDS))) + .build(); + LOG.infof("Raft node '%s' started (%d node(s): %s)", selfNode.id(), cluster.size(), + cluster.stream().map(RaftNode::id).collect(Collectors.joining(", "))); + } + + void stop(@Observes ShutdownEvent event) throws IOException + { + if (client != null) + { + client.close(); + } + if (server != null) + { + server.close(); + } + } + + public String selfId() + { + return selfNode.id(); + } + + public int clusterSize() + { + return group.getPeers().size(); + } + + public void submit(Command command, String resource) + { + Proposal proposal = new Proposal(Instant.now(), auditLogService.principal(), command); + Message message = Message.valueOf(ByteString.copyFrom(commandCodec.encode(proposal))); + RaftClientReply reply = await(client.async().send(message), resource); + if (!metadataStore.awaitApplied(reply.getLogIndex(), timeout)) + { + throw new S3Exception(S3ErrorCode.SERVICE_UNAVAILABLE, resource); + } + String code = reply.getMessage().getContent().toStringUtf8(); + if (!code.isEmpty()) + { + throw new S3Exception(S3ErrorCode.valueOf(code), resource); + } + } + + public T read(Supplier query) + { + if (clusterSize() > 1) + { + barrier(); + } + return query.get(); + } + + public void barrier() + { + try + { + RaftClientReply reply = client.io().sendReadOnly(Message.EMPTY, RaftPeerId.valueOf(selfNode.id())); + if (!reply.isSuccess()) + { + throw new S3Exception(S3ErrorCode.SERVICE_UNAVAILABLE, "metadata", reply.getException()); + } + } + catch (IOException e) + { + throw new S3Exception(S3ErrorCode.SERVICE_UNAVAILABLE, "metadata", e); + } + } + + public boolean awaitReady() + { + boolean ready = false; + while (!ready && !Thread.currentThread().isInterrupted()) + { + ready = leaderReady() && tryBarrier(); + if (!ready) + { + pause(); + } + } + return ready; + } + + public void register() + { + submit(new Command.RegisterNode(selfNode.id(), selfUrl()), selfNode.id()); + } + + private boolean leaderReady() + { + try + { + DivisionInfo info = server.getDivision(GROUP_ID).getInfo(); + if (info.isLeader()) + { + return info.isLeaderReady(); + } + return info.getLeaderId() != null; + } + catch (IOException e) + { + throw new IllegalStateException("Raft group " + GROUP_ID + " is not available", e); + } + } + + private boolean tryBarrier() + { + try + { + barrier(); + return true; + } + catch (S3Exception e) + { + LOG.infof("Waiting for raft cluster to become ready: %s", e.getMessage()); + return false; + } + } + + private void pause() + { + try + { + Thread.sleep(READY_POLL_INTERVAL.toMillis()); + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + } + } + + private String selfUrl() + { + return advertisedUrl.orElse("http://" + selfNode.host() + ":" + httpPort); + } + + private RaftClientReply await(CompletableFuture future, String resource) + { + try + { + RaftClientReply reply = future.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + if (!reply.isSuccess()) + { + throw new S3Exception(S3ErrorCode.SERVICE_UNAVAILABLE, resource, reply.getException()); + } + return reply; + } + catch (InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new S3Exception(S3ErrorCode.SERVICE_UNAVAILABLE, resource, e); + } + catch (ExecutionException | TimeoutException e) + { + future.cancel(true); + throw new S3Exception(S3ErrorCode.SERVICE_UNAVAILABLE, resource, e); + } + } + + private List cluster() + { + List entries = nodes.orElse(List.of()); + if (entries.isEmpty()) + { + return List.of(new RaftNode(SINGLE_NODE_ID, SINGLE_NODE_HOST, port)); + } + return entries.stream().map(String::trim).map(RaftNode::parse).toList(); + } + + private RaftNode resolveSelf(List cluster) + { + if (cluster.size() == 1 && cluster.getFirst().id().equals(SINGLE_NODE_ID)) + { + return cluster.getFirst(); + } + String selfEntry = self.orElseGet(this::hostname); + List matches = cluster.stream().filter(node -> node.matches(selfEntry)).toList(); + if (matches.size() != 1) + { + throw new IllegalStateException("Could not determine this node in dev.dinauer.r9t.raft.nodes=" + nodes.orElse(List.of()) + + " using '" + selfEntry + "'. Set dev.dinauer.r9t.raft.self (DEV_DINAUER_R9T_RAFT_SELF) to this node's host:port entry."); + } + return matches.getFirst(); + } + + private String hostname() + { + String hostname = System.getenv("HOSTNAME"); + if (hostname != null && !hostname.isBlank()) + { + return hostname; + } + try + { + return InetAddress.getLocalHost().getHostName(); + } + catch (UnknownHostException e) + { + throw new IllegalStateException("Could not determine hostname, set dev.dinauer.r9t.raft.self", e); + } + } + + private RaftPeer peer(RaftNode node) + { + return RaftPeer.newBuilder() + .setId(node.id()) + .setAddress(node.host() + ":" + node.port()) + .build(); + } + + private RaftProperties properties() + { + RaftProperties properties = new RaftProperties(); + RaftConfigKeys.Rpc.setType(properties, SupportedRpcType.GRPC); + RaftServerConfigKeys.setStorageDir(properties, List.of(storageDir().toFile())); + GrpcConfigKeys.Server.setPort(properties, selfNode.port()); + GrpcConfigKeys.Server.setHost(properties, bindHost()); + RaftServerConfigKeys.Read.setOption(properties, RaftServerConfigKeys.Read.Option.LINEARIZABLE); + RaftServerConfigKeys.Snapshot.setAutoTriggerEnabled(properties, true); + RaftServerConfigKeys.Snapshot.setAutoTriggerThreshold(properties, SNAPSHOT_THRESHOLD); + RaftServerConfigKeys.Snapshot.setTriggerWhenStopEnabled(properties, true); + RaftServerConfigKeys.Snapshot.setRetentionFileNum(properties, 2); + RaftServerConfigKeys.Log.setPurgeUptoSnapshotIndex(properties, true); + return properties; + } + + private String bindHost() + { + if (selfNode.id().equals(SINGLE_NODE_ID)) + { + return SINGLE_NODE_HOST; + } + return "0.0.0.0"; + } + + private RaftStorage.StartupOption startupOption(boolean existing) + { + if (existing) + { + return RaftStorage.StartupOption.RECOVER; + } + return RaftStorage.StartupOption.FORMAT; + } + + private void verifyMembership() throws IOException + { + Set configured = group.getPeers().stream().map(peer -> peer.getId().toString()).collect(Collectors.toSet()); + Set stored = server.getDivision(GROUP_ID).getRaftConf().getCurrentPeers().stream() + .map(peer -> peer.getId().toString()) + .collect(Collectors.toSet()); + if (!configured.equals(stored)) + { + server.close(); + throw new IllegalStateException("Configured raft nodes " + configured + " do not match the stored cluster membership " + + stored + " in " + storageDir() + ". Changing the node list of an existing cluster is not supported."); + } + } + + private Path storageDir() + { + return Path.of(workdir, ".raft"); + } +} diff --git a/src/main/java/dev/dinauer/r9t/core/s3/DigestCopy.java b/src/main/java/dev/dinauer/r9t/core/s3/DigestCopy.java index ff5303d..978f87c 100644 --- a/src/main/java/dev/dinauer/r9t/core/s3/DigestCopy.java +++ b/src/main/java/dev/dinauer/r9t/core/s3/DigestCopy.java @@ -16,21 +16,27 @@ public final class DigestCopy { } - public static String copyAndHashMd5(InputStream data, Path target) throws IOException + public static Digests copy(InputStream data, Path target) throws IOException { - MessageDigest digest = newMd5(); - try (DigestInputStream digestInputStream = new DigestInputStream(data, digest)) + MessageDigest md5 = newMd5(); + MessageDigest sha256 = newDigest("SHA-256"); + try (InputStream digestInputStream = new DigestInputStream(new DigestInputStream(data, md5), sha256)) { - Files.copy(digestInputStream, target, StandardCopyOption.REPLACE_EXISTING); + long size = Files.copy(digestInputStream, target, StandardCopyOption.REPLACE_EXISTING); + return new Digests(HexFormat.of().formatHex(md5.digest()), HexFormat.of().formatHex(sha256.digest()), size); } - return HexFormat.of().formatHex(digest.digest()); } public static MessageDigest newMd5() + { + return newDigest("MD5"); + } + + private static MessageDigest newDigest(String algorithm) { try { - return MessageDigest.getInstance("MD5"); + return MessageDigest.getInstance(algorithm); } catch (NoSuchAlgorithmException e) { diff --git a/src/main/java/dev/dinauer/r9t/core/s3/Digests.java b/src/main/java/dev/dinauer/r9t/core/s3/Digests.java new file mode 100644 index 0000000..813d853 --- /dev/null +++ b/src/main/java/dev/dinauer/r9t/core/s3/Digests.java @@ -0,0 +1,5 @@ +package dev.dinauer.r9t.core.s3; + +public record Digests(String md5, String sha256, long size) +{ +} diff --git a/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java b/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java deleted file mode 100644 index 51e1428..0000000 --- a/src/main/java/dev/dinauer/r9t/core/s3/FileAttributes.java +++ /dev/null @@ -1,46 +0,0 @@ -package dev.dinauer.r9t.core.s3; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.UserDefinedFileAttributeView; - -public final class FileAttributes -{ - public static final String ETAG = "user.etag"; - public static final String CONTENT_TYPE = "user.contentType"; - public static final String TAGS = "user.tags"; - - private FileAttributes() - { - } - - public static void write(Path file, String name, String value) throws IOException - { - UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); - view.write(name, StandardCharsets.UTF_8.encode(value)); - } - - public static String read(Path file, String name) throws IOException - { - UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); - ByteBuffer buffer = ByteBuffer.allocate(view.size(name)); - view.read(name, buffer); - buffer.flip(); - return StandardCharsets.UTF_8.decode(buffer).toString(); - } - - public static boolean exists(Path file, String name) throws IOException - { - UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); - return view.list().contains(name); - } - - public static void delete(Path file, String name) throws IOException - { - UserDefinedFileAttributeView view = Files.getFileAttributeView(file, UserDefinedFileAttributeView.class); - view.delete(name); - } -} diff --git a/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java b/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java index e60d03b..3db6932 100644 --- a/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java +++ b/src/main/java/dev/dinauer/r9t/core/s3/S3ErrorCode.java @@ -12,6 +12,8 @@ public enum S3ErrorCode INVALID_PART("InvalidPart", 400, "One or more of the specified parts could not be found"), MALFORMED_XML("MalformedXML", 400, "The XML you provided was not well-formed or did not validate against our published schema"), INVALID_TAG("InvalidTag", 400, "The tag provided was not a valid tag. This error can occur if the tag did not pass input validation."), + ACCESS_DENIED("AccessDenied", 403, "Access Denied"), + SERVICE_UNAVAILABLE("ServiceUnavailable", 503, "The service is currently unavailable, please retry"), INTERNAL_ERROR("InternalError", 500, "We encountered an internal error"); private final String code; diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index e864c2d..256d1f0 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -1 +1,2 @@ -quarkus.http.limits.max-body-size=5G \ No newline at end of file +quarkus.http.limits.max-body-size=5G +quarkus.log.category."org.apache.ratis".level=WARN diff --git a/src/test/java/dev/dinauer/r9t/core/metadata/CommandCodecTest.java b/src/test/java/dev/dinauer/r9t/core/metadata/CommandCodecTest.java new file mode 100644 index 0000000..961511a --- /dev/null +++ b/src/test/java/dev/dinauer/r9t/core/metadata/CommandCodecTest.java @@ -0,0 +1,50 @@ +package dev.dinauer.r9t.core.metadata; + +import dev.dinauer.r9t.core.blob.Blob; +import dev.dinauer.r9t.core.s3.Tag; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CommandCodecTest +{ + private static final Instant NOW = Instant.parse("2026-10-10T08:00:00.123456789Z"); + private static final Blob BLOB = new Blob("0123456789abcdef0123456789abcdef", 42, "f".repeat(64)); + private static final List TAGS = List.of(new Tag("env", "prod"), new Tag("ümlaut", "wert mit leerzeichen")); + + private final CommandCodec codec = new CommandCodec(); + + @Test + public void roundTripsEveryCommand() + { + List commands = List.of( + new Command.RegisterNode("node-1_9870", "http://node-1:8080"), + new Command.CreateBucket("bucket", NOW), + new Command.DeleteBucket("bucket"), + new Command.PutBucketTags("bucket", TAGS), + new Command.PutObject("bucket", "a/b/c.txt", "etag", "text/plain", TAGS, NOW, BLOB, "node-1_9870"), + new Command.DeleteObject("bucket", "a/b/c.txt"), + new Command.PutObjectTags("bucket", "a/b/c.txt", List.of()), + new Command.InitiateUpload("upload", "bucket", "key", "application/octet-stream", TAGS, NOW), + new Command.PutPart("upload", 7, "part-etag", BLOB, "node-2_9870"), + new Command.CompleteUpload("upload", "bucket", "key", "etag-2", NOW, BLOB, "node-3_9870"), + new Command.AbortUpload("upload"), + new Command.BlobStored(BLOB.id(), "node-2_9870")); + + for (Command command : commands) + { + Proposal proposal = new Proposal(NOW, "testkey", command); + assertEquals(proposal, codec.decode(codec.encode(proposal))); + } + } + + @Test + public void roundTripsMissingPrincipal() + { + Proposal proposal = new Proposal(NOW, null, new Command.DeleteBucket("bucket")); + assertEquals(proposal, codec.decode(codec.encode(proposal))); + } +} diff --git a/src/test/java/dev/dinauer/r9t/core/metadata/MetadataStoreTest.java b/src/test/java/dev/dinauer/r9t/core/metadata/MetadataStoreTest.java new file mode 100644 index 0000000..e93b60a --- /dev/null +++ b/src/test/java/dev/dinauer/r9t/core/metadata/MetadataStoreTest.java @@ -0,0 +1,164 @@ +package dev.dinauer.r9t.core.metadata; + +import dev.dinauer.r9t.core.blob.Blob; +import dev.dinauer.r9t.core.object.ObjectMetadata; +import dev.dinauer.r9t.core.s3.S3ErrorCode; +import dev.dinauer.r9t.core.s3.Tag; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.time.Instant; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MetadataStoreTest +{ + private static final Instant NOW = Instant.parse("2026-10-10T08:00:00Z"); + private static final String NODE_A = "a_9870"; + private static final String NODE_B = "b_9870"; + + private final MetadataStore store = new MetadataStore(); + private long index; + + @Test + public void overwritingAnObjectReleasesThePreviousBlob() + { + apply(new Command.CreateBucket("bucket", NOW)); + Blob first = blob("1"); + Blob second = blob("2"); + + ApplyResult created = apply(putObject("key", first)); + ApplyResult replaced = apply(putObject("key", second)); + + assertEquals(List.of(first), created.added()); + assertEquals(List.of(first.id()), replaced.released()); + assertEquals(second, store.object("bucket", "key").orElseThrow().blob()); + assertFalse(store.referenced(first.id())); + assertEquals(Set.of(NODE_A), store.holders(second.id())); + } + + @Test + public void validatesAgainstReplicatedState() + { + assertEquals(S3ErrorCode.NO_SUCH_BUCKET, apply(putObject("key", blob("1"))).error()); + apply(new Command.CreateBucket("bucket", NOW)); + assertEquals(S3ErrorCode.BUCKET_ALREADY_EXISTS, apply(new Command.CreateBucket("bucket", NOW)).error()); + apply(putObject("key", blob("1"))); + assertEquals(S3ErrorCode.BUCKET_NOT_EMPTY, apply(new Command.DeleteBucket("bucket")).error()); + assertEquals(S3ErrorCode.NO_SUCH_KEY, apply(new Command.PutObjectTags("bucket", "missing", List.of())).error()); + assertEquals(S3ErrorCode.NO_SUCH_UPLOAD, apply(new Command.AbortUpload("missing")).error()); + } + + @Test + public void tracksBlobHoldersAndReleasesUnknownBlobs() + { + apply(new Command.CreateBucket("bucket", NOW)); + Blob blob = blob("1"); + apply(putObject("key", blob)); + + assertTrue(apply(new Command.BlobStored(blob.id(), NODE_B)).released().isEmpty()); + assertEquals(Set.of(NODE_A, NODE_B), store.holders(blob.id())); + + apply(new Command.DeleteObject("bucket", "key")); + assertEquals(List.of(blob.id()), apply(new Command.BlobStored(blob.id(), NODE_B)).released()); + } + + @Test + public void completingAnUploadReplacesPartsWithTheAssembledBlob() + { + apply(new Command.CreateBucket("bucket", NOW)); + List tags = List.of(new Tag("k", "v")); + apply(new Command.InitiateUpload("upload", "bucket", "key", "text/plain", tags, NOW)); + Blob part1 = blob("1"); + Blob part2 = blob("2"); + Blob replacedPart2 = blob("3"); + Blob assembled = blob("4"); + apply(new Command.PutPart("upload", 1, "e1", part1, NODE_A)); + apply(new Command.PutPart("upload", 2, "e2", part2, NODE_B)); + assertEquals(List.of(part2.id()), apply(new Command.PutPart("upload", 2, "e3", replacedPart2, NODE_A)).released()); + + assertEquals(S3ErrorCode.NO_SUCH_UPLOAD, + apply(new Command.CompleteUpload("upload", "bucket", "other-key", "etag", NOW, assembled, NODE_A)).error()); + ApplyResult completed = apply(new Command.CompleteUpload("upload", "bucket", "key", "etag", NOW, assembled, NODE_A)); + + assertEquals(List.of(part1.id(), replacedPart2.id()), completed.released()); + assertEquals(List.of(assembled), completed.added()); + ObjectMetadata object = store.object("bucket", "key").orElseThrow(); + assertEquals("text/plain", object.contentType()); + assertEquals(tags, object.tags()); + assertTrue(store.upload("upload").isEmpty()); + } + + @Test + public void deletingABucketAbortsItsUploads() + { + apply(new Command.CreateBucket("bucket", NOW)); + apply(new Command.InitiateUpload("upload", "bucket", "key", "text/plain", List.of(), NOW)); + Blob part = blob("1"); + apply(new Command.PutPart("upload", 1, "e1", part, NODE_A)); + + assertEquals(List.of(part.id()), apply(new Command.DeleteBucket("bucket")).released()); + assertTrue(store.upload("upload").isEmpty()); + } + + @Test + public void listsObjectsByPrefix() + { + apply(new Command.CreateBucket("bucket", NOW)); + apply(putObject("a/1", blob("1"))); + apply(putObject("a/2", blob("2"))); + apply(putObject("ab", blob("3"))); + apply(putObject("b", blob("4"))); + + assertEquals(List.of("a/1", "a/2"), store.objects("bucket", "a/").stream().map(ObjectMetadata::key).toList()); + assertEquals(List.of("a/1", "a/2", "ab"), store.objects("bucket", "a").stream().map(ObjectMetadata::key).toList()); + assertEquals(4, store.objects("bucket", null).size()); + } + + @Test + public void snapshotRestoresTheCompleteState() throws Exception + { + apply(new Command.RegisterNode(NODE_A, "http://a:8080")); + apply(new Command.CreateBucket("bucket", NOW)); + apply(new Command.PutBucketTags("bucket", List.of(new Tag("team", "storage")))); + Blob blob = blob("1"); + apply(putObject("key", blob)); + apply(new Command.BlobStored(blob.id(), NODE_B)); + apply(new Command.InitiateUpload("upload", "bucket", "key2", "text/plain", List.of(), NOW)); + apply(new Command.PutPart("upload", 1, "e1", blob("2"), NODE_B)); + + ByteArrayOutputStream snapshot = new ByteArrayOutputStream(); + store.writeSnapshot(snapshot); + MetadataStore restored = new MetadataStore(); + restored.readSnapshot(new ByteArrayInputStream(snapshot.toByteArray()), index); + + assertEquals(store.nodes(), restored.nodes()); + assertEquals(store.buckets(), restored.buckets()); + assertEquals(store.objects("bucket", null), restored.objects("bucket", null)); + assertEquals(store.upload("upload"), restored.upload("upload")); + assertEquals(Set.of(NODE_A, NODE_B), restored.holders(blob.id())); + assertEquals(store.holders(blob("2").id()), restored.holders(blob("2").id())); + assertEquals(index, restored.appliedIndex()); + } + + private ApplyResult apply(Command command) + { + index++; + return store.apply(command, index); + } + + private Command.PutObject putObject(String key, Blob blob) + { + return new Command.PutObject("bucket", key, "etag-" + key, "text/plain", List.of(), NOW, blob, NODE_A); + } + + private Blob blob(String seed) + { + return new Blob(seed.repeat(32).substring(0, 32), seed.length(), seed); + } +} diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 129f0bd..b5b1d34 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -3,3 +3,5 @@ config_ordinal=1000 dev.dinauer.r9t.workdir=target/test-r9t dev.dinauer.r9t.credentials.access-key=testkey dev.dinauer.r9t.credentials.secret-key=testsecret + +dev.dinauer.r9t.raft.port=19870