The pool that emptied when a vendor slowed down

import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;

/**
 * Solution: the pool that emptied when a vendor slowed down.
 *
 * Three defects, and only the first one is a bug. The other two are numbers
 * someone chose while trying to fix the first, and both made it worse — which
 * is the usual shape of a pool incident.
 */
public class Solution {

    /* ── the pool (correct; do not change) ──────────────────────────────── */

    static final class Pool {
        private final Semaphore permits;
        private final AtomicInteger inUse = new AtomicInteger();
        final int size;
        final AtomicInteger acquired = new AtomicInteger();
        final AtomicInteger refused = new AtomicInteger();
        final AtomicLong totalHoldMillis = new AtomicLong();
        final AtomicInteger peakInUse = new AtomicInteger();

        Pool(int size) { this.size = size; this.permits = new Semaphore(size, true); }

        /** Borrow a connection, run `work`, and always give it back. */
        <T> T withConnection(long acquireTimeoutMillis, Callable<T> work) {
            try {
                if (!permits.tryAcquire(acquireTimeoutMillis, TimeUnit.MILLISECONDS)) {
                    refused.incrementAndGet();
                    return null;
                }
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                return null;
            }
            long start = System.nanoTime();
            try {
                peakInUse.accumulateAndGet(inUse.incrementAndGet(), Math::max);
                acquired.incrementAndGet();
                return work.call();
            } catch (Exception e) {
                throw new RuntimeException(e);
            } finally {
                inUse.decrementAndGet();
                totalHoldMillis.addAndGet((System.nanoTime() - start) / 1_000_000);
                permits.release();
            }
        }

        long averageHoldMillis() {
            int n = acquired.get();
            return n == 0 ? 0 : totalHoldMillis.get() / n;
        }
    }

    /** Stands in for the vendor. Slow today. */
    static void chargeCard() { sleep(200); }

    /** A query. Fast, as it has always been. */
    static void runQuery(long millis) { sleep(millis); }

    static void sleep(long millis) {
        try { Thread.sleep(millis); } catch (InterruptedException e) { Thread.currentThread().interrupt(); }
    }

    /* ── the service under review ───────────────────────────────────────── */

    /*
     * Defect 2. Raising the pool to 100 was the response to the incident and
     * it did not help, because a pool is a concurrency limit rather than
     * capacity — past what the database can genuinely execute at once, extra
     * connections add lock contention and cache churn instead of throughput.
     * It also multiplies by the instance count against max_connections.
     *
     * Little's Law gives the real number. With the vendor call moved out
     * (below), a checkout holds a connection for about 40 ms, so 50 requests
     * per second needs 50 * 0.04 = 2 connections. Ten is generous.
     */
    static final int POOL_SIZE = 10;

    /*
     * Defect 3. A thirty-second acquisition timeout means an overloaded
     * service queues for half a minute rather than refusing anything, so
     * requests pile up until callers time out first — and by then the queue is
     * thirty seconds deep and every one of those requests is wasted work.
     *
     * A short timeout sheds load: some requests are refused immediately, which
     * is a better outcome than all of them being slow. This is the same
     * argument as the rate limiter entry, applied to a different queue.
     */
    static final long ACQUIRE_TIMEOUT_MILLIS = 2_000;

    static final class CheckoutService {
        final Pool pool;
        CheckoutService(Pool pool) { this.pool = pool; }

        /*
         * Defect 1, and the only actual bug. The vendor call sat inside the
         * connection's lifetime, so hold time was the vendor's latency plus
         * the queries — and when the vendor's p99 went from 80 ms to 900 ms,
         * hold time went with it. Every endpoint sharing the pool then failed
         * at acquisition, with the database completely idle.
         *
         * Two short transactions with the call between them. The connection is
         * held only for work the database is doing.
         *
         * The order matters: charge first, then record. Recording before
         * charging would mean a crash mid-way leaves an order marked paid that
         * never was. Charging first can leave a charge with no record, which
         * is recoverable from the vendor's side — and an outbox is what
         * removes even that.
         */
        Boolean checkout(long orderId) {
            Boolean loaded = pool.withConnection(ACQUIRE_TIMEOUT_MILLIS, () -> {
                runQuery(20);            // load the order
                return true;
            });
            if (!Boolean.TRUE.equals(loaded)) return null;

            chargeCard();                // outside the connection

            return pool.withConnection(ACQUIRE_TIMEOUT_MILLIS, () -> {
                runQuery(20);            // mark it paid
                return true;
            });
        }

        /** GET /orders/{id} — nothing to do with payments. */
        Boolean viewOrder(long orderId) {
            return pool.withConnection(ACQUIRE_TIMEOUT_MILLIS, () -> {
                runQuery(5);
                return true;
            });
        }
    }

    /* ── checks ─────────────────────────────────────────────────────────── */

    static final int CHECKOUTS = 40;
    static final int VIEWS = 40;

    public static void main(String[] args) throws Exception {
        List<String> failures = new ArrayList<>();

        var pool = new Pool(POOL_SIZE);
        var service = new CheckoutService(pool);
        var done = new CountDownLatch(CHECKOUTS + VIEWS);
        var viewsServed = new AtomicInteger();
        long start = System.nanoTime();

        for (int i = 0; i < CHECKOUTS; i++) daemon(() -> { service.checkout(1); done.countDown(); });
        for (int i = 0; i < VIEWS; i++) daemon(() -> {
            if (Boolean.TRUE.equals(service.viewOrder(1))) viewsServed.incrementAndGet();
            done.countDown();
        });
        done.await(60, TimeUnit.SECONDS);
        long wallMillis = (System.nanoTime() - start) / 1_000_000;

        // 1. A connection must not be held across the vendor call.
        if (pool.averageHoldMillis() > 100)
            failures.add("1. the average connection was held for " + pool.averageHoldMillis()
                       + " ms — the vendor call is inside the connection's lifetime");

        // 2. The pool must be sized from Little's Law, not from optimism.
        if (POOL_SIZE > 25)
            failures.add("2. a pool of " + POOL_SIZE + " pushes more concurrent work at the database "
                       + "than it can run, and multiplies by the instance count");

        // 3. An overloaded service must shed load rather than queue for ages.
        if (ACQUIRE_TIMEOUT_MILLIS > 5_000)
            failures.add("3. an acquisition timeout of " + ACQUIRE_TIMEOUT_MILLIS
                       + " ms queues instead of shedding load — callers give up first");

        /*
         * There is deliberately no check on whether the order views succeeded.
         * A pool of 100 has enough permits for this load, so they all do — and
         * a real database would be struggling at 100 concurrent connections in
         * a way this model does not represent. Asserting on it here would be
         * asserting on a property of the model rather than of the code.
         *
         * The isolation failure is real, though: when the pool IS the right
         * size, a vendor call held inside a connection starves every endpoint
         * that shares the pool. Check 1 is what prevents it.
         */
        System.out.println("  (wall time " + wallMillis + " ms, peak connections " + pool.peakInUse.get()
                         + ", views served " + viewsServed.get() + "/" + VIEWS
                         + ", refused " + pool.refused.get() + ")");

        /* ── report ─────────────────────────────────────────────────────── */
        if (failures.isEmpty()) {
            System.out.println("PASS");
        } else {
            failures.forEach(f -> System.out.println("  " + f));
            System.out.println("FAIL");
        }
    }

    static void daemon(Runnable body) {
        var t = new Thread(body);
        t.setDaemon(true);
        t.start();
    }
}