ExerciseProduction incident
Production incident
The cache that existed twice
40 minintermediate2–10 yrs
A real incident: symptom first, cause hidden, tradeoff at the end.
The incident
A process-wide settings cache, written three years ago and reviewed by two
people at the time.
A report then ran against a value that had been updated an hour earlier,
and the investigation found two caches alive in one JVM.
What the team found:
1. Nobody could say which of three possible causes it was, because all
three are present in the class.
2. The application serializes objects into a distributed session store.
3. A test framework on the classpath constructs objects reflectively.
4. The class also exposes its map directly, which is a fourth problem and
not a singleton problem at all.
Close every door. The fix for three of them is one construct; the fourth
needs a separate decision.
What this teaches
- Check-then-act construction races once per JVM, at startup
- setAccessible walks past a private constructor; only an enum refuses
- Deserialization builds a new object unless readResolve exists
- An enum closes all three with no code from you
- Constructing a singleton correctly says nothing about using it correctly
Starter
Starter.java
import java.io.*;
import java.lang.reflect.*;
import java.util.*;
import java.util.concurrent.*;
/**
* Production: the cache that existed twice.
*
* A process-wide settings cache. It has been in production for three years
* and was reviewed by two people when it was written.
*
* Then a report ran against a stale value that had been updated an hour
* earlier, and the investigation found two caches in one JVM. Nobody could
* say which of three possible causes it was, because all three are present.
*
* Run it. Four checks, four failures.
*/
public class Starter {
/* ─────────── the singleton under review ─────────── */
static class SettingsCache implements Serializable {
private static SettingsCache instance;
/** Public, mutable, and reachable from every thread in the process. */
public Map<String, String> settings = new HashMap<>();
private SettingsCache() { }
public static SettingsCache get() {
if (instance == null) {
instance = new SettingsCache();
}
return instance;
}
}
static final int ROUNDS = 200, THREADS = 8;
public static void main(String[] args) throws Exception {
boolean ok = true;
System.out.println("── construction under concurrent first use ──");
int leaked = raceRounds();
System.out.println(" rounds with several instances : " + leaked + " of " + ROUNDS);
ok &= check("concurrent first use produces one instance", leaked == 0);
System.out.println();
System.out.println("── the other two ways in ──");
SettingsCache one = SettingsCache.get();
boolean reflectionBlocked;
try {
// Try the ordinary no-arg constructor, then the signature every
// enum constant has — so a refusal is a real refusal and not just
// a missing overload.
Constructor<?> ctor;
Object[] args2;
try {
ctor = SettingsCache.class.getDeclaredConstructor();
args2 = new Object[0];
} catch (NoSuchMethodException e) {
ctor = SettingsCache.class.getDeclaredConstructor(String.class, int.class);
args2 = new Object[] { "HACK", 1 };
}
ctor.setAccessible(true);
Object other = ctor.newInstance(args2);
reflectionBlocked = (other == one);
System.out.println(" reflection : produced " + (reflectionBlocked ? "the same" : "a SECOND") + " instance");
} catch (Exception e) {
reflectionBlocked = true;
System.out.println(" reflection : refused — " + e.getClass().getSimpleName()
+ (e.getMessage() == null ? "" : " — " + e.getMessage()));
}
ok &= check("reflection cannot produce a second instance", reflectionBlocked);
SettingsCache copy = roundTrip(one);
System.out.println(" serialization : returned " + (copy == one ? "the same" : "a SECOND") + " instance");
ok &= check("a serialization round trip returns the same instance", copy == one);
System.out.println();
System.out.println("── what it exposes ──");
List<String> exposed = mutableFields(SettingsCache.class);
System.out.println(" mutable or public fields : " + exposed);
ok &= check("the singleton exposes no mutable state", exposed.isEmpty());
System.out.println();
System.out.println(ok ? "PASS" : "FAIL");
}
/**
* The static field can only be raced once per JVM, so it is reset between
* rounds to make a startup-only bug observable.
*/
static int raceRounds() throws Exception {
Field field;
try {
field = SettingsCache.class.getDeclaredField("instance");
field.setAccessible(true);
} catch (NoSuchFieldException e) {
field = null; // no field to reset: nothing to race
}
int leaked = 0;
for (int round = 0; round < ROUNDS; round++) {
if (field != null) field.set(null, null);
Set<Object> seen = ConcurrentHashMap.newKeySet();
CountDownLatch start = new CountDownLatch(1);
Thread[] threads = new Thread[THREADS];
for (int i = 0; i < THREADS; i++) {
threads[i] = new Thread(() -> {
try { start.await(); } catch (InterruptedException e) { return; }
for (int n = 0; n < 50; n++) seen.add(SettingsCache.get());
});
threads[i].start();
}
start.countDown();
for (Thread t : threads) t.join();
if (seen.size() > 1) leaked++;
}
return leaked;
}
/** Non-static fields that are public or non-final — shared mutable state. */
static List<String> mutableFields(Class<?> type) {
List<String> out = new ArrayList<>();
for (Field f : type.getDeclaredFields()) {
int m = f.getModifiers();
if (Modifier.isStatic(m)) continue;
if (Modifier.isPublic(m) || !Modifier.isFinal(m)) out.add(f.getName());
}
return out;
}
@SuppressWarnings("unchecked")
static <T extends Serializable> T roundTrip(T value) throws Exception {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { out.writeObject(value); }
try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
return (T) in.readObject();
}
}
static boolean check(String what, boolean passed) {
System.out.println((passed ? " ok " : " FAIL ") + what);
return passed;
}
}Run it locally:
cd exercises/java/creational-patterns/thread-safe-singleton/03-production
javac Starter.java -d /tmp/out && java -cp /tmp/out StarterHints
Hint 1
Three of the four checks fail deterministically. Read each failure message before changing anything.
Hint 2
Which of the three construction defects can a class-based singleton NOT be protected from?
Hint 3
The race is reset between rounds on purpose. Ask why that was necessary, and what it says about finding this bug in production.
Hint 4
The fourth check is about the field, not the pattern.
Done when
- Concurrent first use produces one instance across every round
- Reflection is refused, and the refusal message says why
- A serialization round trip returns the same instance
- The singleton exposes no mutable state
- A comment names which defect no class-based form can fix
Solution
Show the solution — try it yourself first
Solution.java
import java.io.*;
import java.lang.reflect.*;
import java.util.*;
import java.util.concurrent.*;
/**
* Solution: the cache that existed twice.
*
* Three separate ways in, and the enum closes all of them at once.
*
* 1. Construction. `if (instance == null) instance = new ...` is
* check-then-act, so two threads at first use both construct. It can
* only happen once per JVM, at startup, which is why three years passed
* without it — and why no test found it.
*
* 2. Reflection. A private constructor is not a boundary: setAccessible
* walks past it. Constructor.newInstance refuses enums explicitly, so
* this is the one defect that CANNOT be fixed in a class-based version.
*
* 3. Serialization. Deserializing a Serializable singleton builds a new
* object every time, silently. A class needs readResolve, and needs
* someone to remember it forever. Enums are deserialized by name.
*
* A holder class fixes 1 and readResolve fixes 3, and nothing fixes 2. That
* is the entire argument for the enum: one construct, no code, all three.
*
* The fourth check is not about the pattern at all. A public mutable HashMap
* on a process-wide object is shared mutable state whatever created it, so
* the field becomes private and final over a concurrent map. Constructing a
* singleton correctly and using it correctly are separate problems.
*/
public class Solution {
/* ─────────── the singleton under review ─────────── */
enum SettingsCache {
INSTANCE;
/** Private, final, and concurrent — the contents are shared too. */
private final Map<String, String> settings = new ConcurrentHashMap<>();
public static SettingsCache get() {
return INSTANCE;
}
public Map<String, String> settings() {
return settings;
}
}
static final int ROUNDS = 200, THREADS = 8;
public static void main(String[] args) throws Exception {
boolean ok = true;
System.out.println("── construction under concurrent first use ──");
int leaked = raceRounds();
System.out.println(" rounds with several instances : " + leaked + " of " + ROUNDS);
ok &= check("concurrent first use produces one instance", leaked == 0);
System.out.println();
System.out.println("── the other two ways in ──");
SettingsCache one = SettingsCache.get();
boolean reflectionBlocked;
try {
// Try the ordinary no-arg constructor, then the signature every
// enum constant has — so a refusal is a real refusal and not just
// a missing overload.
Constructor<?> ctor;
Object[] args2;
try {
ctor = SettingsCache.class.getDeclaredConstructor();
args2 = new Object[0];
} catch (NoSuchMethodException e) {
ctor = SettingsCache.class.getDeclaredConstructor(String.class, int.class);
args2 = new Object[] { "HACK", 1 };
}
ctor.setAccessible(true);
Object other = ctor.newInstance(args2);
reflectionBlocked = (other == one);
System.out.println(" reflection : produced " + (reflectionBlocked ? "the same" : "a SECOND") + " instance");
} catch (Exception e) {
reflectionBlocked = true;
System.out.println(" reflection : refused — " + e.getClass().getSimpleName()
+ (e.getMessage() == null ? "" : " — " + e.getMessage()));
}
ok &= check("reflection cannot produce a second instance", reflectionBlocked);
SettingsCache copy = roundTrip(one);
System.out.println(" serialization : returned " + (copy == one ? "the same" : "a SECOND") + " instance");
ok &= check("a serialization round trip returns the same instance", copy == one);
System.out.println();
System.out.println("── what it exposes ──");
List<String> exposed = mutableFields(SettingsCache.class);
System.out.println(" mutable or public fields : " + exposed);
ok &= check("the singleton exposes no mutable state", exposed.isEmpty());
System.out.println();
System.out.println(ok ? "PASS" : "FAIL");
}
/**
* The static field can only be raced once per JVM, so it is reset between
* rounds to make a startup-only bug observable.
*/
static int raceRounds() throws Exception {
Field field;
try {
field = SettingsCache.class.getDeclaredField("instance");
field.setAccessible(true);
} catch (NoSuchFieldException e) {
field = null; // no field to reset: nothing to race
}
int leaked = 0;
for (int round = 0; round < ROUNDS; round++) {
if (field != null) field.set(null, null);
Set<Object> seen = ConcurrentHashMap.newKeySet();
CountDownLatch start = new CountDownLatch(1);
Thread[] threads = new Thread[THREADS];
for (int i = 0; i < THREADS; i++) {
threads[i] = new Thread(() -> {
try { start.await(); } catch (InterruptedException e) { return; }
for (int n = 0; n < 50; n++) seen.add(SettingsCache.get());
});
threads[i].start();
}
start.countDown();
for (Thread t : threads) t.join();
if (seen.size() > 1) leaked++;
}
return leaked;
}
/** Non-static fields that are public or non-final — shared mutable state. */
static List<String> mutableFields(Class<?> type) {
List<String> out = new ArrayList<>();
for (Field f : type.getDeclaredFields()) {
int m = f.getModifiers();
if (Modifier.isStatic(m)) continue;
if (Modifier.isPublic(m) || !Modifier.isFinal(m)) out.add(f.getName());
}
return out;
}
@SuppressWarnings("unchecked")
static <T extends Serializable> T roundTrip(T value) throws Exception {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try (ObjectOutputStream out = new ObjectOutputStream(bytes)) { out.writeObject(value); }
try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(bytes.toByteArray()))) {
return (T) in.readObject();
}
}
static boolean check(String what, boolean passed) {
System.out.println((passed ? " ok " : " FAIL ") + what);
return passed;
}
}Stretch
The fix makes the cache an enum, which cannot extend a class and cannot take
constructor arguments from configuration. Write the version this application
should actually have — a Spring bean — and say what changes about testing,
about the number of instances, and about what "singleton" even means once a
second application context exists.