Overhaul of SlaveLaunchLogs

pom.xml

@@ -67,7 +67,8 @@
 
   <properties>
     <changelist>999999-SNAPSHOT</changelist>
-    <jenkins.version>2.361.4</jenkins.version>
+    <!-- TODO https://github.com/jenkinsci/jenkins/pull/9009 -->
+    <jenkins.version>2.448-rc34683.5cd5ff72b_b_89</jenkins.version>
     <gitHubRepo>jenkinsci/${project.artifactId}-plugin</gitHubRepo>
     <spotless.check.skip>false</spotless.check.skip>
   </properties>

src/main/java/com/cloudbees/jenkins/support/api/LaunchLogsFileContent.java

@@ -1,6 +1,7 @@
 package com.cloudbees.jenkins.support.api;
 
 import com.cloudbees.jenkins.support.filter.PasswordRedactor;
+import com.cloudbees.jenkins.support.impl.SlaveLaunchLogs;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
@@ -11,6 +12,9 @@
 import java.util.stream.Collectors;
 import org.apache.commons.io.IOUtils;
 
+/**
+ * @see SlaveLaunchLogs
+ */
 public class LaunchLogsFileContent extends FileContent {
 
     public LaunchLogsFileContent(String name, String[] filterableParameters, File file, long maxSize) {

src/main/java/com/cloudbees/jenkins/support/impl/SlaveLaunchLogs.java

@@ -33,18 +33,28 @@
 import com.cloudbees.jenkins.support.timer.FileListCapComponent;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import hudson.Extension;
+import hudson.ExtensionList;
+import hudson.console.ConsoleLogFilter;
+import hudson.console.LineTransformationOutputStream;
 import hudson.model.AbstractModelObject;
 import hudson.model.Computer;
-import hudson.model.Node;
+import hudson.model.Run;
 import hudson.security.Permission;
-import hudson.slaves.Cloud;
+import hudson.slaves.SlaveComputer;
+import hudson.triggers.SafeTimerTask;
+import hudson.util.io.RewindableRotatingFileOutputStream;
 import java.io.File;
-import java.util.ArrayList;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import jenkins.model.Jenkins;
+import org.apache.commons.io.output.TeeOutputStream;
 import org.jenkinsci.Symbol;
 import org.kohsuke.stapler.DataBoundConstructor;
 
@@ -54,6 +64,9 @@
 @Extension
 public class SlaveLaunchLogs extends ObjectComponent<Computer> {
 
+    private static final int MAX_ROTATE_LOGS =
+            Integer.getInteger(SlaveLaunchLogs.class.getName() + ".MAX_ROTATE_LOGS", 9);
+
     @DataBoundConstructor
     public SlaveLaunchLogs() {
         super();
@@ -73,128 +86,80 @@
 
     @Override
     public void addContents(@NonNull Container container) {
-        addAgentsLaunchLogs(container);
+        File log = ExtensionList.lookupSingleton(LogArchiver.class).log;
+        if (log.isFile()) {
+            // TODO perhaps include rotated files as well
+            container.add(new LaunchLogsFileContent(
+                    "nodes/slave/launchLogs.log", new String[] {}, log, FileListCapComponent.MAX_FILE_SIZE));
+        }
     }
 
     @NonNull
     @Override
     public ComponentCategory getCategory() {
-        return ComponentCategory.AGENT; // TODO or LOGS?
+        return ComponentCategory.LOGS;
     }
 
     @Override
     public void addContents(@NonNull Container container, Computer item) {
-        if (item.getNode() == null) {
-            return;
-        }
-        File lastLog = item.getLogFile();
-        if (lastLog.exists()) {
-            Agent agent = new Agent(new File(Jenkins.get().getRootDir(), "logs/slaves/" + item.getName()), lastLog);
-            if (!agent.isTooOld()) {
-                addAgentLaunchLogs(container, agent);
-            }
-        }
-    }
-
-    /**
-     * <p>
-     * In the presence of {@link Cloud} plugins like EC2, we want to find past agents, not just current ones.
-     * So we don't try to loop through {@link Node} here but just try to look at the file systems to find them
-     * all.
-     *
-     * <p>
-     * Generally these cloud plugins do not clean up old logs, so if run for a long time, the log directory
-     * will be full of old files that are not very interesting. Use some heuristics to cut off logs
-     * that are old.
-     */
-    private void addAgentsLaunchLogs(Container result) {
-
-        List<Agent> all = new ArrayList<>();
-
-        { // find all the agent launch log files and sort them newer ones first
-            File agentLogsDir = new File(Jenkins.get().getRootDir(), "logs/slaves");
-            File[] logs = agentLogsDir.listFiles();
-            if (logs != null) {
-                for (File dir : logs) {
-                    File lastLog = new File(dir, "slave.log");
-                    if (lastLog.exists()) {
-                        Agent s = new Agent(dir, lastLog);
-                        if (s.isTooOld()) continue; // we don't care
-                        all.add(s);
-                    }
+        if (item.getNode() != null
+                && item.getLogFile().lastModified() >= System.currentTimeMillis() - TimeUnit.DAYS.toMillis(7)) {
+            File dir = new File(Jenkins.get().getRootDir(), "logs/slaves/" + item.getName());
+            File[] files = dir.listFiles(ROTATED_LOGFILE_FILTER);
+            if (files != null) {
+                for (File f : files) {
+                    container.add(new LaunchLogsFileContent(
+                            "nodes/slave/{0}/launchLogs/{1}",
+                            new String[] {dir.getName(), f.getName()}, f, FileListCapComponent.MAX_FILE_SIZE));
                 }
             }
-
-            Collections.sort(all);
-        }
-        { // this might be still too many, so try to cap them.
-            int acceptableSize = Math.max(256, Jenkins.get().getNodes().size() * 5);
-
-            if (all.size() > acceptableSize) all = all.subList(0, acceptableSize);
-        }
-
-        // now add them all
-        all.forEach(it -> addAgentLaunchLogs(result, it));
-    }
-
-    private void addAgentLaunchLogs(Container container, Agent agent) {
-        File[] files = agent.dir.listFiles(ROTATED_LOGFILE_FILTER);
-        if (files != null) {
-            for (File f : files) {
-                container.add(new LaunchLogsFileContent(
-                        "nodes/slave/{0}/launchLogs/{1}",
-                        new String[] {agent.getName(), f.getName()}, f, FileListCapComponent.MAX_FILE_SIZE));
-            }
         }
     }
 
-    static class Agent implements Comparable<Agent> {
-        /**
-         * Launch log directory of the agent: logs/slaves/NAME
-         */
-        final File dir;
-
-        final long time;
-
-        Agent(File dir, File lastLog) {
-            this.dir = dir;
-            this.time = lastLog.lastModified();
-        }
+    @Extension
+    public static final class LogArchiver extends ConsoleLogFilter {
 
-        /** Agent name */
-        String getName() {
-            return dir.getName();
-        }
+        final File log = new File(SafeTimerTask.getLogsRoot(), "nodeLaunchLogs.txt");
+        private final RewindableRotatingFileOutputStream stream =
+                new RewindableRotatingFileOutputStream(log, MAX_ROTATE_LOGS);
 
-        /**
-         * Use the primary log file's timestamp to compare newer agents from older agents.
-         *
-         * sort in descending order; newer ones first.
-         */
-        public int compareTo(Agent that) {
-            return Long.compare(that.time, this.time);
+        @Override
+        public OutputStream decorateLogger(Computer computer, OutputStream logger) {
+            if (computer instanceof SlaveComputer) {
+                return new TeeOutputStream(logger, new PrefixedStream(stream, computer.getName()));
+            } else {
+                return logger;
+            }
         }
 
+        @SuppressWarnings("rawtypes")
         @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
+        public OutputStream decorateLogger(Run build, OutputStream logger) throws IOException, InterruptedException {
+            return logger;
+        }
+    }
 
-            Agent agent = (Agent) o;
+    static class PrefixedStream extends LineTransformationOutputStream.Delegating {
+        private final String name;
 
-            return time == agent.time;
+        PrefixedStream(OutputStream out, String name) {
+            super(out);
+            this.name = name;
         }
 
         @Override
-        public int hashCode() {
-            return (int) (time ^ (time >>> 32));
-        }
-
-        /**
-         * If the file is more than 7 days old, it is considered too old.
-         */
-        public boolean isTooOld() {
-            return time < System.currentTimeMillis() - TimeUnit.DAYS.toMillis(7);
+        protected void eol(byte[] b, int len) throws IOException {
+            synchronized (out) {
+                out.write('[');
+                out.write(DateTimeFormatter.ISO_INSTANT
+                        .format(Instant.now().truncatedTo(ChronoUnit.MILLIS))
+                        .getBytes(StandardCharsets.US_ASCII));
+                out.write(' ');
+                out.write(name.getBytes(StandardCharsets.UTF_8));
+                out.write(']');
+                out.write(' ');
+                out.write(b, 0, len);
+            }
         }
     }
 

src/test/java/com/cloudbees/jenkins/support/Security2186Test.java

@@ -75,8 +75,7 @@ public void secretsFilterWhenSystemPropertyContainsPasswordThenValueRedacted() t
         verifyFileIfContainsPassword(zip, "nodes/master/system.properties", "test2186.trustStorePassword");
         verifyFileIfContainsPassword(zip, "nodes/master/environment.txt", "test2186.trustStorePassword");
         verifyFileIfContainsPassword(zip, "nodes/slave/slave0/config.xml", "-Dtest2186.trustStoreAgentPassword");
-        verifyFileIfContainsPassword(
-                zip, "nodes/slave/slave0/launchLogs/slave.log", "-Dtest2186.trustStoreAgentPassword");
+        verifyFileIfContainsPassword(zip, "nodes/slave/launchLogs.log", "-Dtest2186.trustStoreAgentPassword");
     }
 
     private void verifyFileIfContainsPassword(ZipFile zip, String fileName, String expectedPasswordKey)