The redeploy that ate the host

import java.io.*;
import java.lang.ref.*;
import java.util.*;

/**
 * Solution: the redeploy that ate the host.
 *
 * A class loader is released all at once or not at all. It references every
 * class it loaded, each of those references it back, and every instance
 * references its class — so one reference from outside pins the entire
 * deploy: all its classes, all their static fields, and everything those
 * reach.
 *
 * Two things were holding on here, both added for good reasons:
 *
 *   DEPLOY_HISTORY kept the Class object so an admin console could list what
 *   had been deployed. A Class is a hard reference to its loader.
 *
 *   ACTIVE_PLUGIN is a ThreadLocal, so the value stays attached to whichever
 *   thread ran the deploy — and in a server that is a pooled thread which
 *   never dies. Same mechanism as the ThreadLocal leak in
 *   /java/jvm/memory-leaks, with a class loader on the end of it instead of
 *   a tenant id.
 *
 * Dropping the loader variable released neither, which is the lesson:
 * "nothing points at the loader" is not the same as "nothing points at
 * anything the loader loaded".
 *
 * WHY THE HEAP DUMP SHOWED NOTHING
 *   What is retained is class metadata, and since Java 8 that lives in
 *   Metaspace — native memory, outside -Xmx and outside a heap dump. The
 *   heap really was healthy. `jcmd <pid> VM.classloader_stats` would have
 *   shown four live loaders in a process that had deployed four times,
 *   which is the whole diagnosis in one command.
 *
 *   Under Java 7 the same leak filled a fixed PermGen and threw
 *   OutOfMemoryError: PermGen space within the hour. Removing the ceiling
 *   did not fix the bug, it removed the alarm.
 */
public class Solution {

    /** The "plugin" class. Each deploy loads its own private copy of this. */
    public static class Plugin {
        public static String describe() {
            return "plugin";
        }
    }

    static final String PLUGIN = "Solution$Plugin";

    /**
     * A deploy's class loader. It loads Plugin itself rather than delegating,
     * which is what gives each deploy an independent copy — and what makes
     * the loader worth collecting once the deploy is superseded.
     */
    static final class DeployLoader extends ClassLoader {
        final int version;

        DeployLoader(ClassLoader parent, int version) {
            super(parent);
            this.version = version;
        }

        @Override
        protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
            if (!name.equals(PLUGIN)) {
                return super.loadClass(name, resolve);
            }
            synchronized (getClassLoadingLock(name)) {
                Class<?> already = findLoadedClass(name);
                if (already != null) {
                    return already;
                }
                try (InputStream in = getParent().getResourceAsStream(PLUGIN + ".class")) {
                    byte[] bytes = in.readAllBytes();
                    return defineClass(name, bytes, 0, bytes.length);
                } catch (IOException e) {
                    throw new ClassNotFoundException(name, e);
                }
            }
        }
    }

    /* ── the two things that hold on, both added for good reasons ── */

    /** Added so the admin console could list what had ever been deployed. */
    static final List<Class<?>> DEPLOY_HISTORY = new ArrayList<>();

    /** Added so a request could find the plugin handling it. */
    static final ThreadLocal<Class<?>> ACTIVE_PLUGIN = new ThreadLocal<>();

    static DeployLoader deploy(int version) throws Exception {
        DeployLoader loader = new DeployLoader(Solution.class.getClassLoader(), version);
        Class<?> plugin = loader.loadClass(PLUGIN);
        plugin.getMethod("describe").invoke(null);

        DEPLOY_HISTORY.add(plugin);
        ACTIVE_PLUGIN.set(plugin);
        return loader;
    }

    /**
     * FIX: release everything that reaches into the deploy, not just the
     * loader variable.
     *
     * Keeping the admin console working is still possible — it just cannot
     * keep Class objects. Record the name and the version, which are plain
     * strings and hold nothing.
     */
    static void undeploy(int version) {
        for (Class<?> deployed : DEPLOY_HISTORY) {
            HISTORY_NAMES.add("v" + version + " " + deployed.getName());
        }
        DEPLOY_HISTORY.clear();
        ACTIVE_PLUGIN.remove();
    }

    /** What the admin console actually needed: names, not Class objects. */
    static final List<String> HISTORY_NAMES = new ArrayList<>();

    public static void main(String[] args) throws Exception {
        List<WeakReference<ClassLoader>> probes = new ArrayList<>();

        System.out.println("── four deploys ──");
        for (int version = 1; version <= 4; version++) {
            DeployLoader loader = deploy(version);
            probes.add(new WeakReference<>(loader));
            undeploy(version);
            loader = null;
            System.out.println("  deployed and superseded v" + version);
        }

        int released = 0;
        for (WeakReference<ClassLoader> probe : probes) {
            if (collected(probe)) released++;
        }

        System.out.println();
        System.out.println("history kept, holding nothing : " + HISTORY_NAMES.size() + " entries");
        System.out.println();
        System.out.println("deploys made              : " + probes.size());
        System.out.println("class loaders released    : " + released);
        System.out.println("still pinned in memory    : " + (probes.size() - released));
        System.out.println();
        System.out.println("every superseded deploy released : " + (released == probes.size()));
        System.out.println(released == probes.size() ? "PASS" : "FAIL");
    }

    /**
     * System.gc() is a hint, so asking once proves nothing. This asks
     * repeatedly with allocation pressure in between, and gives up after a
     * bounded number of attempts so a leak cannot hang the build.
     */
    static boolean collected(WeakReference<?> ref) {
        for (int attempt = 0; attempt < 50 && ref.get() != null; attempt++) {
            System.gc();
            byte[] churn = new byte[1 << 20];
            if (churn.length < 0) System.out.print("");
        }
        return ref.get() == null;
    }
}