diff --git a/pom.xml b/pom.xml index b8fe5cc..e145acb 100644 --- a/pom.xml +++ b/pom.xml @@ -77,6 +77,10 @@ io.quarkus quarkus-mutiny + + io.quarkus + quarkus-rest-client-jackson + diff --git a/src/main/java/dev/dinauer/inspect/log/LogWebsocket.java b/src/main/java/dev/dinauer/inspect/log/LogWebsocket.java index 64a7162..b2245d5 100644 --- a/src/main/java/dev/dinauer/inspect/log/LogWebsocket.java +++ b/src/main/java/dev/dinauer/inspect/log/LogWebsocket.java @@ -97,9 +97,9 @@ public class LogWebsocket if (indexFirstSpace != -1) { String timestampRaw = log.substring(0, indexFirstSpace); - String message = log.substring(indexFirstSpace).trim(); try { + String message = log.substring(indexFirstSpace + 1); result.add(new KubernetesLog(LocalDateTime.parse(timestampRaw, DateTimeFormatter.ISO_DATE_TIME), message)); } catch (Exception e) diff --git a/src/main/java/dev/dinauer/monitoring/MonitoringService.java b/src/main/java/dev/dinauer/monitoring/MonitoringService.java index 50c2905..b8dc55c 100644 --- a/src/main/java/dev/dinauer/monitoring/MonitoringService.java +++ b/src/main/java/dev/dinauer/monitoring/MonitoringService.java @@ -23,7 +23,7 @@ public class MonitoringService { case LABEL -> { - return podService.findByLabels(targetConfig.getNamespace(), targetConfig.getLabels()).stream().filter(pod -> pod.getStatus().getPhase().equals("Running")).toList(); + return podService.findByNamespaceAndLabels(targetConfig.getNamespace(), targetConfig.getLabels()).stream().filter(pod -> pod.getStatus().getPhase().equals("Running")).toList(); } case DEPLOYMENT, STATEFUL_SET -> { diff --git a/src/main/java/dev/dinauer/monitoring/TopNodesService.java b/src/main/java/dev/dinauer/monitoring/TopNodesService.java index 82fd964..09914ac 100644 --- a/src/main/java/dev/dinauer/monitoring/TopNodesService.java +++ b/src/main/java/dev/dinauer/monitoring/TopNodesService.java @@ -2,6 +2,9 @@ package dev.dinauer.monitoring; import dev.dinauer.ProcessRunner; import dev.dinauer.monitoring.nodes.MonitoredNode; +import dev.dinauer.monitoring.nodes.NodeMetrics; +import dev.dinauer.monitoring.nodes.client.NodeDiskMetrics; +import dev.dinauer.monitoring.nodes.client.NodeDiskService; import dev.dinauer.service.PodService; import dev.dinauer.utils.ClientProvider; import io.fabric8.kubernetes.api.model.Node; @@ -23,12 +26,16 @@ public class TopNodesService @Inject PodService podService; + @Inject + NodeDiskService nodeDiskService; + public List findAll() { List result = new ArrayList<>(); List stats = runTopNodesCommand(); Map podsOnNodes = countPods(); + Map nodeDiskMetrics = nodeDiskService.getDiskMetrics(); for(String nodeName : stats) { String[] parts = nodeName.split("\\s+"); @@ -40,7 +47,20 @@ public class TopNodesService Integer relativeCpu = extractInteger(parts[2]); Integer absoluteMemory = extractMemory(parts[3]); Integer relativeMemory = extractInteger(parts[4]); - result.add(new MonitoredNode(node, absoluteCpu, relativeCpu, Integer.parseInt(node.getStatus().getAllocatable().get("cpu").getAmount()) * 1000, absoluteMemory, relativeMemory, extractMemory(node.getStatus().getAllocatable().get("memory").getAmount()), podsOnNodes.get(node.getMetadata().getName()))); + Integer totalCpu = Integer.parseInt(node.getStatus().getAllocatable().get("cpu").getAmount()) * 1000; + Integer totalMemory = extractMemory(node.getStatus().getAllocatable().get("memory").getAmount()); + Integer totalPods = podsOnNodes.get(node.getMetadata().getName()); + NodeDiskMetrics diskMetrics = nodeDiskMetrics.get(node.getMetadata().getName()); + if (diskMetrics != null) + { + NodeMetrics metrics = new NodeMetrics(absoluteCpu, relativeCpu, totalCpu, absoluteMemory, relativeMemory, totalMemory, totalPods, diskMetrics.relativeDiskUsage(), diskMetrics.totalDiskSpace()); + result.add(new MonitoredNode(node, metrics)); + } + else + { + NodeMetrics metrics = new NodeMetrics(absoluteCpu, relativeCpu, totalCpu, absoluteMemory, relativeMemory, totalMemory, totalPods, null, null); + result.add(new MonitoredNode(node, metrics)); + } } } return result; diff --git a/src/main/java/dev/dinauer/monitoring/nodes/MonitoredNode.java b/src/main/java/dev/dinauer/monitoring/nodes/MonitoredNode.java index 2a58c16..dd4d166 100644 --- a/src/main/java/dev/dinauer/monitoring/nodes/MonitoredNode.java +++ b/src/main/java/dev/dinauer/monitoring/nodes/MonitoredNode.java @@ -4,58 +4,16 @@ import io.fabric8.kubernetes.api.model.Node; public class MonitoredNode extends Node { - public MonitoredNode(Node node, Integer absoluteCpuUsage, Integer relativeCpuUsage, Integer totalCpu, Integer absoluteMemory, Integer relativeMemory, Integer totalMemory, Integer runningPods) + public MonitoredNode(Node node, NodeMetrics metrics) { super(node.getApiVersion(), node.getKind(), node.getMetadata(), node.getSpec(), node.getStatus()); - this.absoluteCpuUsage = absoluteCpuUsage; - this.relativeCpuUsage = relativeCpuUsage; - this.totalCpu = totalCpu; - this.absoluteMemory = absoluteMemory; - this.relativeMemory = relativeMemory; - this.totalMemory = totalMemory; - this.runningPods = runningPods; + this.metrics = metrics; } - private final Integer absoluteCpuUsage; - private final Integer relativeCpuUsage; - private final Integer totalCpu; - private final Integer absoluteMemory; - private final Integer relativeMemory; - private final Integer totalMemory; - private final Integer runningPods; + private final NodeMetrics metrics; - public Integer getAbsoluteCpuUsage() + public NodeMetrics getMetrics() { - return absoluteCpuUsage; - } - - public Integer getRelativeCpuUsage() - { - return relativeCpuUsage; - } - - public Integer getTotalCpu() - { - return totalCpu; - } - - public Integer getAbsoluteMemory() - { - return absoluteMemory; - } - - public Integer getRelativeMemory() - { - return relativeMemory; - } - - public Integer getTotalMemory() - { - return totalMemory; - } - - public Integer getRunningPods() - { - return runningPods; + return metrics; } } \ No newline at end of file diff --git a/src/main/java/dev/dinauer/monitoring/nodes/NodeMetrics.java b/src/main/java/dev/dinauer/monitoring/nodes/NodeMetrics.java new file mode 100644 index 0000000..b757493 --- /dev/null +++ b/src/main/java/dev/dinauer/monitoring/nodes/NodeMetrics.java @@ -0,0 +1,5 @@ +package dev.dinauer.monitoring.nodes; + +public record NodeMetrics(Integer absoluteCpuUsage, Integer relativeCpuUsage, Integer totalCpu, Integer absoluteMemory, Integer relativeMemory, Integer totalMemory, Integer runningPods, Integer relativeDiskUsage, Long totalDiskSpace) +{ +} diff --git a/src/main/java/dev/dinauer/monitoring/nodes/NodeMonitoringService.java b/src/main/java/dev/dinauer/monitoring/nodes/NodeMonitoringService.java index 92469f7..6d41fc1 100644 --- a/src/main/java/dev/dinauer/monitoring/nodes/NodeMonitoringService.java +++ b/src/main/java/dev/dinauer/monitoring/nodes/NodeMonitoringService.java @@ -22,12 +22,24 @@ public class NodeMonitoringService List nodes = topNodesService.findAll(); for (MonitoredNode node : nodes) { + NodeMetrics nodeMetrics = node.getMetrics(); Map metrics = Map.ofEntries( - Map.entry("RELATIVE_CPU", (long) node.getRelativeCpuUsage()), - Map.entry("RELATIVE_MEMORY", (long) node.getRelativeMemory()), - Map.entry("ABSOLUTE_MEMORY", (long) node.getAbsoluteMemory()), - Map.entry("ABSOLUTE_CPU", (long) node.getAbsoluteCpuUsage())); + Map.entry("RELATIVE_CPU", toLong(nodeMetrics.relativeCpuUsage())), + Map.entry("RELATIVE_MEMORY", toLong(nodeMetrics.relativeMemory())), + Map.entry("ABSOLUTE_MEMORY", toLong(nodeMetrics.absoluteMemory())), + Map.entry("ABSOLUTE_CPU", toLong(nodeMetrics.absoluteCpuUsage())), + Map.entry("RELATIVE_DISK_SPACE", toLong(nodeMetrics.relativeDiskUsage())), + Map.entry("TOTAL_DISK_SPACE", toLong(nodeMetrics.totalDiskSpace()))); indexingService.index(String.format("NODE-%s", node.getMetadata().getUid()), "NODE_METRICS", metrics); } } + + private Long toLong(Integer input) + { + if (input != null) + { + return Long.valueOf(input); + } + return null; + } } diff --git a/src/main/java/dev/dinauer/monitoring/nodes/client/NodeDiskMetrics.java b/src/main/java/dev/dinauer/monitoring/nodes/client/NodeDiskMetrics.java new file mode 100644 index 0000000..152e0a0 --- /dev/null +++ b/src/main/java/dev/dinauer/monitoring/nodes/client/NodeDiskMetrics.java @@ -0,0 +1,5 @@ +package dev.dinauer.monitoring.nodes.client; + +public record NodeDiskMetrics(Integer relativeDiskUsage, Long totalDiskSpace) +{ +} diff --git a/src/main/java/dev/dinauer/monitoring/nodes/client/NodeDiskService.java b/src/main/java/dev/dinauer/monitoring/nodes/client/NodeDiskService.java new file mode 100644 index 0000000..a63f095 --- /dev/null +++ b/src/main/java/dev/dinauer/monitoring/nodes/client/NodeDiskService.java @@ -0,0 +1,72 @@ +package dev.dinauer.monitoring.nodes.client; + +import dev.dinauer.service.PodService; +import io.fabric8.kubernetes.api.model.Pod; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.jboss.logging.Logger; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@ApplicationScoped +public class NodeDiskService +{ + private static final Map label = Map.ofEntries(Map.entry("dev.dinauer.kubooboo/component", "node-monitor")); + private static final Integer PORT = 8080; + + @Inject + Logger LOG; + + @Inject + PodService podService; + + public Map getDiskMetrics() + { + Map result = new HashMap<>(); + List pods = podService.findByLabels(label); + for (Pod pod : pods) + { + String nodeName = pod.getSpec().getNodeName(); + String ip = pod.getStatus().getPodIP(); + LOG.infof("Collect disk monitoring for node %s", nodeName); + try (HttpClient client = HttpClient.newBuilder().build()) + { + HttpRequest request = HttpRequest.newBuilder().uri(new URI(String.format("http://%s:%s", ip, PORT))).GET().build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + NodeDiskMetrics metrics = parse(response.body()); + result.put(nodeName, metrics); + } + catch (URISyntaxException | IOException | InterruptedException e) + { + LOG.errorf("Failed to collect disk monitoring for node %s", nodeName); + } + } + return result; + } + + private NodeDiskMetrics parse(String input) + { + Map result = new HashMap<>(); + for (String line : input.split("\\s+")) + { + String[] sections = line.split(":"); + if (sections.length == 2) + { + result.put(sections[0], sections[1]); + } + else + { + LOG.errorf("Cannot parse metrics line '%s'", line); + } + } + return new NodeDiskMetrics(Integer.parseInt(result.get("percentage_used")), Long.parseLong(result.get("total-space"))); + } +} diff --git a/src/main/java/dev/dinauer/service/NodeService.java b/src/main/java/dev/dinauer/service/NodeService.java index eea74a6..0876f96 100644 --- a/src/main/java/dev/dinauer/service/NodeService.java +++ b/src/main/java/dev/dinauer/service/NodeService.java @@ -2,7 +2,6 @@ package dev.dinauer.service; import dev.dinauer.monitoring.TopNodesService; import dev.dinauer.monitoring.nodes.MonitoredNode; -import io.fabric8.kubernetes.api.model.Node; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; import jakarta.enterprise.context.ApplicationScoped; diff --git a/src/main/java/dev/dinauer/service/PodService.java b/src/main/java/dev/dinauer/service/PodService.java index 752720d..089cb13 100644 --- a/src/main/java/dev/dinauer/service/PodService.java +++ b/src/main/java/dev/dinauer/service/PodService.java @@ -2,13 +2,13 @@ package dev.dinauer.service; import dev.dinauer.utils.ClientProvider; import io.fabric8.kubernetes.api.model.Pod; +import io.fabric8.kubernetes.api.model.apps.DaemonSet; import io.fabric8.kubernetes.api.model.apps.StatefulSet; import io.fabric8.kubernetes.client.Watch; import io.fabric8.kubernetes.client.Watcher; import io.fabric8.kubernetes.client.dsl.AppsAPIGroupDSL; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import org.jboss.resteasy.reactive.common.NotImplementedYet; import java.util.List; import java.util.Map; @@ -57,17 +57,22 @@ public class PodService implements ResourceService StatefulSet set = apps.statefulSets().inNamespace(namespace).withName(name).get(); if (set != null) { - return findByLabels(namespace, set.getSpec().getSelector().getMatchLabels()); + return findByNamespaceAndLabels(namespace, set.getSpec().getSelector().getMatchLabels()); } return null; } } - public List findByLabels(String namespace, Map labels) + public List findByNamespaceAndLabels(String namespace, Map labels) { return clientProvider.getClient().pods().inNamespace(namespace).withLabels(labels).list().getItems(); } + public List findByLabels(Map labels) + { + return clientProvider.getClient().pods().inAnyNamespace().withLabels(labels).list().getItems(); + } + public List findAll() { return clientProvider.getClient().pods().inAnyNamespace().list().getItems(); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index a39bd7a..6677d57 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -5,6 +5,7 @@ quarkus.http.root-path=/api dev.dinauer.kubooboo.work.dir=/var/lib/kubooboo/work %dev.dev.dinauer.kubooboo.work.dir=/home/andreas/Documents/dev/kubooboo/backend/src/main/resources/dev +dev.dinauer.kubooboo.current.namespace=${CURRENT_NAMESPACE} # Keys %prod.smallrye.jwt.sign.key.location=${PRIVATE_KEY_LOCATION}