✨ New Metrics

pom.xml

@@ -77,6 +77,10 @@
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-mutiny</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-rest-client-jackson</artifactId>
+        </dependency>
 
         <!-- Hibernate ORM specific dependencies -->
         <dependency>

src/main/java/dev/dinauer/inspect/log/LogWebsocket.java

@@ -97,9 +97,9 @@ public class LogWebsocket
             if (indexFirstSpace != -1)
             {
                 String timestampRaw = log.substring(0, indexFirstSpace);
-                String message = log.substring(indexFirstSpace).trim();
                 try
                 {
+                    String message = log.substring(indexFirstSpace + 1);
                     result.add(new KubernetesLog(LocalDateTime.parse(timestampRaw, DateTimeFormatter.ISO_DATE_TIME), message));
                 }
                 catch (Exception e)

src/main/java/dev/dinauer/monitoring/MonitoringService.java

@@ -23,7 +23,7 @@ public class MonitoringService
         {
             case LABEL ->
             {
-                return podService.findByLabels(targetConfig.getNamespace(), targetConfig.getLabels()).stream().filter(pod -> pod.getStatus().getPhase().equals("Running")).toList();
+                return podService.findByNamespaceAndLabels(targetConfig.getNamespace(), targetConfig.getLabels()).stream().filter(pod -> pod.getStatus().getPhase().equals("Running")).toList();
             }
             case DEPLOYMENT, STATEFUL_SET ->
             {

src/main/java/dev/dinauer/monitoring/TopNodesService.java

@@ -2,6 +2,9 @@ package dev.dinauer.monitoring;
 
 import dev.dinauer.ProcessRunner;
 import dev.dinauer.monitoring.nodes.MonitoredNode;
+import dev.dinauer.monitoring.nodes.NodeMetrics;
+import dev.dinauer.monitoring.nodes.client.NodeDiskMetrics;
+import dev.dinauer.monitoring.nodes.client.NodeDiskService;
 import dev.dinauer.service.PodService;
 import dev.dinauer.utils.ClientProvider;
 import io.fabric8.kubernetes.api.model.Node;
@@ -23,12 +26,16 @@ public class TopNodesService
     @Inject
     PodService podService;
 
+    @Inject
+    NodeDiskService nodeDiskService;
+
     public List<MonitoredNode> findAll()
     {
         List<MonitoredNode> result = new ArrayList<>();
 
         List<String> stats = runTopNodesCommand();
         Map<String, Integer> podsOnNodes = countPods();
+        Map<String, NodeDiskMetrics> nodeDiskMetrics = nodeDiskService.getDiskMetrics();
         for(String nodeName : stats)
         {
             String[] parts = nodeName.split("\\s+");
@@ -40,7 +47,20 @@ public class TopNodesService
                 Integer relativeCpu = extractInteger(parts[2]);
                 Integer absoluteMemory = extractMemory(parts[3]);
                 Integer relativeMemory = extractInteger(parts[4]);
-                result.add(new MonitoredNode(node, absoluteCpu, relativeCpu, Integer.parseInt(node.getStatus().getAllocatable().get("cpu").getAmount()) * 1000, absoluteMemory, relativeMemory, extractMemory(node.getStatus().getAllocatable().get("memory").getAmount()), podsOnNodes.get(node.getMetadata().getName())));
+                Integer totalCpu = Integer.parseInt(node.getStatus().getAllocatable().get("cpu").getAmount()) * 1000;
+                Integer totalMemory = extractMemory(node.getStatus().getAllocatable().get("memory").getAmount());
+                Integer totalPods = podsOnNodes.get(node.getMetadata().getName());
+                NodeDiskMetrics diskMetrics = nodeDiskMetrics.get(node.getMetadata().getName());
+                if (diskMetrics != null)
+                {
+                    NodeMetrics metrics = new NodeMetrics(absoluteCpu, relativeCpu, totalCpu, absoluteMemory, relativeMemory, totalMemory, totalPods, diskMetrics.relativeDiskUsage(), diskMetrics.totalDiskSpace());
+                    result.add(new MonitoredNode(node, metrics));
+                }
+                else
+                {
+                    NodeMetrics metrics = new NodeMetrics(absoluteCpu, relativeCpu, totalCpu, absoluteMemory, relativeMemory, totalMemory, totalPods, null, null);
+                    result.add(new MonitoredNode(node, metrics));
+                }
             }
         }
         return result;

src/main/java/dev/dinauer/monitoring/nodes/MonitoredNode.java

@@ -4,58 +4,16 @@ import io.fabric8.kubernetes.api.model.Node;
 
 public class MonitoredNode extends Node
 {
-    public MonitoredNode(Node node, Integer absoluteCpuUsage, Integer relativeCpuUsage, Integer totalCpu, Integer absoluteMemory, Integer relativeMemory, Integer totalMemory, Integer runningPods)
+    public MonitoredNode(Node node, NodeMetrics metrics)
     {
         super(node.getApiVersion(), node.getKind(), node.getMetadata(), node.getSpec(), node.getStatus());
-        this.absoluteCpuUsage = absoluteCpuUsage;
+        this.metrics = metrics;
-        this.relativeCpuUsage = relativeCpuUsage;
-        this.totalCpu = totalCpu;
-        this.absoluteMemory = absoluteMemory;
-        this.relativeMemory = relativeMemory;
-        this.totalMemory = totalMemory;
-        this.runningPods = runningPods;
     }
 
-    private final Integer absoluteCpuUsage;
+    private final NodeMetrics metrics;
-    private final Integer relativeCpuUsage;
-    private final Integer totalCpu;
-    private final Integer absoluteMemory;
-    private final Integer relativeMemory;
-    private final Integer totalMemory;
-    private final Integer runningPods;
 
-    public Integer getAbsoluteCpuUsage()
+    public NodeMetrics getMetrics()
     {
-        return absoluteCpuUsage;
+        return metrics;
-    }
-
-    public Integer getRelativeCpuUsage()
-    {
-        return relativeCpuUsage;
-    }
-
-    public Integer getTotalCpu()
-    {
-        return totalCpu;
-    }
-
-    public Integer getAbsoluteMemory()
-    {
-        return absoluteMemory;
-    }
-
-    public Integer getRelativeMemory()
-    {
-        return relativeMemory;
-    }
-
-    public Integer getTotalMemory()
-    {
-        return totalMemory;
-    }
-
-    public Integer getRunningPods()
-    {
-        return runningPods;
     }
 }

src/main/java/dev/dinauer/monitoring/nodes/NodeMetrics.java

@@ -0,0 +1,5 @@
+package dev.dinauer.monitoring.nodes;
+
+public record NodeMetrics(Integer absoluteCpuUsage, Integer relativeCpuUsage, Integer totalCpu, Integer absoluteMemory, Integer relativeMemory, Integer totalMemory, Integer runningPods, Integer relativeDiskUsage, Long totalDiskSpace)
+{
+}

src/main/java/dev/dinauer/monitoring/nodes/NodeMonitoringService.java

@@ -22,12 +22,24 @@ public class NodeMonitoringService
         List<MonitoredNode> nodes = topNodesService.findAll();
         for (MonitoredNode node : nodes)
         {
+            NodeMetrics nodeMetrics = node.getMetrics();
             Map<String, Long> metrics = Map.ofEntries(
-                Map.entry("RELATIVE_CPU", (long) node.getRelativeCpuUsage()),
+                Map.entry("RELATIVE_CPU", toLong(nodeMetrics.relativeCpuUsage())),
-                Map.entry("RELATIVE_MEMORY", (long) node.getRelativeMemory()),
+                Map.entry("RELATIVE_MEMORY", toLong(nodeMetrics.relativeMemory())),
-                Map.entry("ABSOLUTE_MEMORY", (long) node.getAbsoluteMemory()),
+                Map.entry("ABSOLUTE_MEMORY", toLong(nodeMetrics.absoluteMemory())),
-                Map.entry("ABSOLUTE_CPU", (long) node.getAbsoluteCpuUsage()));
+                Map.entry("ABSOLUTE_CPU", toLong(nodeMetrics.absoluteCpuUsage())),
+                Map.entry("RELATIVE_DISK_SPACE", toLong(nodeMetrics.relativeDiskUsage())),
+                Map.entry("TOTAL_DISK_SPACE", toLong(nodeMetrics.totalDiskSpace())));
             indexingService.index(String.format("NODE-%s", node.getMetadata().getUid()), "NODE_METRICS", metrics);
         }
     }
+
+    private Long toLong(Integer input)
+    {
+        if (input != null)
+        {
+            return Long.valueOf(input);
+        }
+        return null;
+    }
 }

src/main/java/dev/dinauer/monitoring/nodes/client/NodeDiskMetrics.java

@@ -0,0 +1,5 @@
+package dev.dinauer.monitoring.nodes.client;
+
+public record NodeDiskMetrics(Integer relativeDiskUsage, Long totalDiskSpace)
+{
+}

src/main/java/dev/dinauer/monitoring/nodes/client/NodeDiskService.java

@@ -0,0 +1,72 @@
+package dev.dinauer.monitoring.nodes.client;
+
+import dev.dinauer.service.PodService;
+import io.fabric8.kubernetes.api.model.Pod;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@ApplicationScoped
+public class NodeDiskService
+{
+    private static final Map<String, String> label = Map.ofEntries(Map.entry("dev.dinauer.kubooboo/component", "node-monitor"));
+    private static final Integer PORT = 8080;
+
+    @Inject
+    Logger LOG;
+
+    @Inject
+    PodService podService;
+
+    public Map<String, NodeDiskMetrics> getDiskMetrics()
+    {
+        Map<String, NodeDiskMetrics> result = new HashMap<>();
+        List<Pod> pods = podService.findByLabels(label);
+        for (Pod pod : pods)
+        {
+            String nodeName = pod.getSpec().getNodeName();
+            String ip = pod.getStatus().getPodIP();
+            LOG.infof("Collect disk monitoring for node %s", nodeName);
+            try (HttpClient client = HttpClient.newBuilder().build())
+            {
+                HttpRequest request = HttpRequest.newBuilder().uri(new URI(String.format("http://%s:%s", ip, PORT))).GET().build();
+                HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
+                NodeDiskMetrics metrics = parse(response.body());
+                result.put(nodeName, metrics);
+            }
+            catch (URISyntaxException | IOException | InterruptedException e)
+            {
+                LOG.errorf("Failed to collect disk monitoring for node %s", nodeName);
+            }
+        }
+        return result;
+    }
+
+    private NodeDiskMetrics parse(String input)
+    {
+        Map<String, String> result = new HashMap<>();
+        for (String line : input.split("\\s+"))
+        {
+            String[] sections = line.split(":");
+            if (sections.length == 2)
+            {
+                result.put(sections[0], sections[1]);
+            }
+            else
+            {
+                LOG.errorf("Cannot parse metrics line '%s'", line);
+            }
+        }
+        return new NodeDiskMetrics(Integer.parseInt(result.get("percentage_used")), Long.parseLong(result.get("total-space")));
+    }
+}

src/main/java/dev/dinauer/service/NodeService.java

@@ -2,7 +2,6 @@ package dev.dinauer.service;
 
 import dev.dinauer.monitoring.TopNodesService;
 import dev.dinauer.monitoring.nodes.MonitoredNode;
-import io.fabric8.kubernetes.api.model.Node;
 import io.fabric8.kubernetes.client.Watch;
 import io.fabric8.kubernetes.client.Watcher;
 import jakarta.enterprise.context.ApplicationScoped;

src/main/java/dev/dinauer/service/PodService.java

@@ -2,13 +2,13 @@ package dev.dinauer.service;
 
 import dev.dinauer.utils.ClientProvider;
 import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.api.model.apps.DaemonSet;
 import io.fabric8.kubernetes.api.model.apps.StatefulSet;
 import io.fabric8.kubernetes.client.Watch;
 import io.fabric8.kubernetes.client.Watcher;
 import io.fabric8.kubernetes.client.dsl.AppsAPIGroupDSL;
 import jakarta.enterprise.context.ApplicationScoped;
 import jakarta.inject.Inject;
-import org.jboss.resteasy.reactive.common.NotImplementedYet;
 
 import java.util.List;
 import java.util.Map;
@@ -57,17 +57,22 @@ public class PodService implements ResourceService<Pod>
             StatefulSet set = apps.statefulSets().inNamespace(namespace).withName(name).get();
             if (set != null)
             {
-                return findByLabels(namespace, set.getSpec().getSelector().getMatchLabels());
+                return findByNamespaceAndLabels(namespace, set.getSpec().getSelector().getMatchLabels());
             }
             return null;
         }
     }
 
-    public List<Pod> findByLabels(String namespace, Map<String, String> labels)
+    public List<Pod> findByNamespaceAndLabels(String namespace, Map<String, String> labels)
     {
         return clientProvider.getClient().pods().inNamespace(namespace).withLabels(labels).list().getItems();
     }
 
+    public List<Pod> findByLabels(Map<String, String> labels)
+    {
+        return clientProvider.getClient().pods().inAnyNamespace().withLabels(labels).list().getItems();
+    }
+
     public List<Pod> findAll()
     {
         return clientProvider.getClient().pods().inAnyNamespace().list().getItems();

src/main/resources/application.properties

@@ -5,6 +5,7 @@ quarkus.http.root-path=/api
 
 dev.dinauer.kubooboo.work.dir=/var/lib/kubooboo/work
 %dev.dev.dinauer.kubooboo.work.dir=/home/andreas/Documents/dev/kubooboo/backend/src/main/resources/dev
+dev.dinauer.kubooboo.current.namespace=${CURRENT_NAMESPACE}
 
 # Keys
 %prod.smallrye.jwt.sign.key.location=${PRIVATE_KEY_LOCATION}