Local models, scored against exact answers

Can a local model write a correct simulation?

Each model running on this Mac was given the plain-English rules of a family of questions and asked to write a Python simulation, once, with no retries. Its code ran in a sandbox with no network and no access to anything private. Its answers on two test cases were compared with the exact answers: within 4 standard errors counts as agreeing.

A model that disagrees has usually read a rule differently, which is exactly the mistake an independent check exists to catch. Read its code below to see where.

ModelScoreDice
dice
Secret Santa
selfdraw
Secret Santa
hatdraw
Collecting
collector
gemma4:26b4 of 4agreesagreesagreesagrees
gpt-oss:20b4 of 4agreesagreesagreesagrees
qwen3.6:27b-mlx4 of 4agreesagreesagreesagrees
qwen2.5-coder:7b1 of 4disagreesdisagreeserroragrees

Dice: dice

The rules each model was given
Roll `dice` fair six-sided dice. If they all show the same face, you have won.
Otherwise keep every die showing your most common face (if two faces tie for most common,
keep either one; it does not change the odds) and re-roll all the others. You may roll at
most `rolls` times in total (the first roll counts as one). Report the probability that
at some point all `dice` dice show the same face.
gemma4:26b · agrees · 10.7 s to write
  • {"dice": 5, "rolls": 3}: returned 0.046, exact 0.0460286, z = -0.06
  • {"dice": 4, "rolls": 2}: returned 0.03927, exact 0.0387088, z = +1.30
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    """
    Performs a Monte Carlo simulation to estimate the probability of 
    achieving all dice showing the same face within a given number of rolls.
    """
    dice_count = params["dice"]
    max_rolls = params["rolls"]
    
    rng = random.Random(seed)
    wins = 0

    # Pre-calculate range for efficiency
    roll_range = range(1, 7)

    for _ in range(games):
        # Initial roll
        current_dice = [rng.randint(1, 6) for _ in range(dice_count)]
        
        won = False
        # Check if first roll is already a win
        first_val = current_dice[0]
        is_win = True
        for i in range(1, dice_count):
            if current_dice[i] != first_val:
                is_win = False
                break
        
        if is_win:
            wins += 1
            continue

        # Subsequent rolls
        for roll_num in range(2, max_rolls + 1):
            # Count frequencies to find the most common face
            counts = {}
            for d in current_dice:
                counts[d] = counts.get(d, 0) + 1
            
            # Find the face with the maximum frequency
            # If there's a tie, the first one encountered is chosen
            best_face = -1
            max_freq = -1
            for face, freq in counts.items():
                if freq > max_freq:
                    max_freq = freq
                    best_face = face
            
            # Number of dice to re-roll
            dice_to_reroll = dice_count - max_freq
            
            # Update current_dice: keep best_face, re-roll others
            # We replace the 'others' with new rolls
            new_dice = [best_face] * max_freq
            for _ in range(dice_to_reroll):
                new_dice.append(rng.randint(1, 6))
            current_dice = new_dice
            
            # Check if all dice are now the same
            all_same = True
            target = current_dice[0]
            for i in range(1, dice_count):
                if current_dice[i] != target:
                    all_same = False
                    break
            
            if all_same:
                won = True
                break
        
        if won:
            wins += 1

    return {"p": wins / games}
gpt-oss:20b · agrees · 517.6 s to write
  • {"dice": 5, "rolls": 3}: returned 0.046765, exact 0.0460286, z = +1.57
  • {"dice": 4, "rolls": 2}: returned 0.03837, exact 0.0387088, z = -0.79
import random
from typing import Dict

def simulate(params: Dict[str, int], games: int, seed: int) -> Dict[str, float]:
    """
    Monte‑Carlo simulation of the dice‑matching game.

    Parameters
    ----------
    params : dict
        Dictionary with keys:
            - 'dice' : number of dice to roll
            - 'rolls': maximum number of rolls allowed (first roll counts)
    games : int
        Number of independent games to simulate.
    seed : int
        Seed for the random number generator.

    Returns
    -------
    dict
        Dictionary with a single key 'p' containing the estimated probability
        that all dice show the same face at some point during the game.
    """
    dice = params.get("dice", 0)
    rolls = params.get("rolls", 0)

    if dice <= 0 or rolls <= 0 or games <= 0:
        return {"p": 0.0}

    rng = random.Random(seed)
    successes = 0

    # Pre‑allocate a list for the dice values
    dice_vals = [0] * dice

    for _ in range(games):
        # First roll
        for i in range(dice):
            dice_vals[i] = rng.randint(1, 6)

        # Check for immediate win
        if all(v == dice_vals[0] for v in dice_vals):
            successes += 1
            continue

        # Subsequent rolls
        for _ in range(rolls - 1):
            # Count occurrences of each face (1‑6)
            counts = [0] * 7  # index 0 unused
            for v in dice_vals:
                counts[v] += 1

            # Determine the most common face
            max_face = 1
            max_count = counts[1]
            for face in range(2, 7):
                if counts[face] > max_count:
                    max_count = counts[face]
                    max_face = face

            # Keep dice showing max_face, re‑roll the rest
            for i in range(dice):
                if dice_vals[i] != max_face:
                    dice_vals[i] = rng.randint(1, 6)

            # Check for win
            if all(v == dice_vals[0] for v in dice_vals):
                successes += 1
                break

    return {"p": successes / games}
qwen2.5-coder:7b · disagrees · 8.2 s to write
  • {"dice": 5, "rolls": 3}: returned 0.04371, exact 0.0460286, z = -4.95
  • {"dice": 4, "rolls": 2}: returned 0.06886, exact 0.0387088, z = +69.90
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    dice = params['dice']
    rolls = params['rolls']
    rng = random.Random(seed)
    
    def roll_dice():
        return [rng.randint(1, 6) for _ in range(dice)]
    
    def most_common_face(dice):
        from collections import Counter
        count = Counter(dice)
        max_count = max(count.values())
        return [face for face, cnt in count.items() if cnt == max_count][0]
    
    def simulate_game():
        dice = roll_dice()
        for _ in range(rolls - 1):
            if len(set(dice)) == 1:
                return True
            most_common = most_common_face(dice)
            dice = [most_common if x != most_common else rng.randint(1, 6) for x in dice]
        return len(set(dice)) == 1
    
    wins = sum(simulate_game() for _ in range(games))
    probability = wins / games
    return {"p": probability}
qwen3.6:27b-mlx · agrees · 1043.1 s to write
  • {"dice": 5, "rolls": 3}: returned 0.04661, exact 0.0460286, z = +1.24
  • {"dice": 4, "rolls": 2}: returned 0.03867, exact 0.0387088, z = -0.09
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    rng = random.Random(seed)
    n_dice = params["dice"]
    max_rolls = params["rolls"]
    
    wins = 0
    faces = (1, 2, 3, 4, 5, 6)
    
    for _ in range(games):
        kept = 0
        current_face = 1
        won = False
        
        for _ in range(max_rolls):
            to_roll = n_dice - kept
            
            new_rolls = rng.choices(faces, k=to_roll)
            
            counts = [0] * 7
            for d in new_rolls:
                counts[d] += 1
                
            counts[current_face] += kept
            
            max_count = 0
            best_face = 1
            for f in range(1, 7):
                c = counts[f]
                if c > max_count:
                    max_count = c
                    best_face = f
                    
            if max_count == n_dice:
                won = True
                break
                
            kept = max_count
            current_face = best_face
            
        if won:
            wins += 1
            
    return {"p": wins / games}

Secret Santa: selfdraw

The rules each model was given
`people` people each put their name in a hat. The names are shuffled uniformly at random
and each person takes one (equivalently: a uniformly random permutation of the people).
Nobody puts a name back. Report the probability that at least one person ends up with
their own name.
gemma4:26b · agrees · 4.6 s to write
  • {"people": 10}: returned 0.631785, exact 0.632121, z = -0.31
  • {"people": 4}: returned 0.623795, exact 0.625, z = -1.11
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    """
    Performs a Monte Carlo simulation to estimate the probability of at least 
    one fixed point in a random permutation (the derangement problem).
    """
    n = params["people"]
    rng = random.Random(seed)
    
    # Pre-create a list of indices to shuffle. 
    # Shuffling a list of integers 0 to n-1 is equivalent to 
    # shuffling names and checking if index i matches value i.
    indices = list(range(n))
    
    successes = 0
    
    for _ in range(games):
        rng.shuffle(indices)
        
        # Check if any element is in its original position
        # We use a generator expression with any() for speed and early exit
        if any(indices[i] == i for i in range(n)):
            successes += 1
            
    return {"p": successes / games}
gpt-oss:20b · agrees · 433.0 s to write
  • {"people": 10}: returned 0.633435, exact 0.632121, z = +1.22
  • {"people": 4}: returned 0.6236, exact 0.625, z = -1.29
import random
from typing import Dict

def simulate(params: Dict, games: int, seed: int) -> Dict[str, float]:
    """
    Monte‑Carlo simulation of the classic matching problem.

    Parameters
    ----------
    params : dict
        Must contain the key 'people' with an integer value.
    games : int
        Number of independent games to simulate.
    seed : int
        Seed for the random number generator.

    Returns
    -------
    dict
        Dictionary with a single key 'p' containing the estimated probability
        that at least one person gets their own name.
    """
    people = int(params.get("people", 0))
    rng = random.Random(seed)

    # Reuse a single list for the permutation to avoid repeated allocations.
    indices = list(range(people))
    successes = 0

    for _ in range(games):
        rng.shuffle(indices)          # random permutation of indices
        # Check for a fixed point: i == indices[i]
        for i, val in enumerate(indices):
            if i == val:
                successes += 1
                break

    return {"p": successes / games}
qwen2.5-coder:7b · disagrees · 4.9 s to write
  • {"people": 10}: returned 0, exact 0.632121, z = -586.22
  • {"people": 4}: returned 0.042005, exact 0.625, z = -538.55
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    people = params['people']
    rng = random.Random(seed)
    
    def derangement(n):
        if n == 0:
            return 1
        if n == 1:
            return 0
        return (n - 1) * (derangement(n - 1) + derangement(n - 2))
    
    total_games = games
    successful_games = 0
    
    for _ in range(games):
        name_list = list(range(people))
        rng.shuffle(name_list)
        if name_list == list(range(people)):
            successful_games += 1
    
    probability = successful_games / total_games
    return {"p": probability}

# Example usage:
# result = simulate({"people": 10}, 100000, 42)
# print(result)
qwen3.6:27b-mlx · agrees · 94.9 s to write
  • {"people": 10}: returned 0.633435, exact 0.632121, z = +1.22
  • {"people": 4}: returned 0.6236, exact 0.625, z = -1.29
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    rng = random.Random(seed)
    n = params["people"]
    successes = 0
    base = list(range(n))
    for _ in range(games):
        rng.shuffle(base)
        for i in range(n):
            if base[i] == i:
                successes += 1
                break
    return {"p": successes / games}

Secret Santa: hatdraw

The rules each model was given
`people` people (numbered 0..people-1) put their names in a hat and then draw one at a time,
in order 0, 1, 2, ... Each person draws a name; if it is a name they are not allowed to have,
they put it back and draw again, so in effect they take a uniformly random name among the
allowed names still in the hat. Names taken are not returned. A person may never take their
own name. If `couples` is true, the people form couples (0,1), (2,3), (4,5), ... and a person
may also never take their partner's name. The draw FAILS if some person, at their turn, finds
that every name left in the hat is one they are not allowed to have. Report the probability
that the draw fails.
gemma4:26b · agrees · 12.8 s to write
  • {"people": 8, "couples": false}: returned 0.091135, exact 0.0904946, z = +1.00
  • {"people": 10, "couples": true}: returned 0.247835, exact 0.24761, z = +0.23
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    """
    Performs a Monte Carlo simulation to estimate the probability of a failed draw.
    
    Rules:
    - People 0 to N-1 draw names one by one.
    - A person cannot draw their own name.
    - If couples=True, person i cannot draw their partner (i^1).
    - If a person's turn comes and all remaining names in the hat are forbidden, the draw fails.
    - Names are not returned once drawn.
    """
    people_count = params["people"]
    couples_enabled = params["couples"]
    rng = random.Random(seed)
    
    # Pre-calculate forbidden partners for each person to speed up the loop
    # forbidden[i] is a set of indices person i cannot pick.
    forbidden = []
    for i in range(people_count):
        f = {i}
        if couples_enabled:
            # Partner of i is i ^ 1 (0<->1, 2<->3, etc.)
            f.add(i ^ 1)
        forbidden.append(f)

    failures = 0
    
    # Optimization: Use a list for the hat and track available indices.
    # Since we need to pick a random element from 'allowed' names,
    # we can maintain a list of names currently in the hat.
    
    for _ in range(games):
        # The 'hat' contains all indices currently available
        hat = list(range(people_count))
        failed_game = False
        
        for i in range(people_count):
            # Identify which names in the current hat are allowed for person i
            # We iterate through the hat to find valid candidates.
            # To keep it fast, we filter the hat.
            
            # Find indices in 'hat' that are not in forbidden[i]
            # We use a list comprehension for speed in CPython
            allowed_indices_in_hat = [idx for idx in hat if idx not in forbidden[i]]
            
            if not allowed_indices_in_hat:
                failed_game = True
                break
            
            # Pick a random name from the allowed names
            # We need to pick one from the 'allowed_indices_in_hat' 
            # and then remove that specific value from the 'hat'
            chosen_name = rng.choice(allowed_indices_in_hat)
            
            # Remove the chosen name from the hat
            # Note: list.remove() is O(N), but N is small (up to people_count)
            hat.remove(chosen_name)
            
        if failed_game:
            failures += 1
            
    return {"p": failures / games}

gpt-oss:20b · agrees · 310.1 s to write
  • {"people": 8, "couples": false}: returned 0.09105, exact 0.0904946, z = +0.87
  • {"people": 10, "couples": true}: returned 0.246885, exact 0.24761, z = -0.75
import random
from typing import Dict

def simulate(params: Dict[str, object], games: int, seed: int) -> Dict[str, float]:
    """
    Monte‑Carlo simulation of the hat‑drawing process described in the problem.
    
    Parameters
    ----------
    params : dict
        Dictionary containing:
            - "people" : int, number of participants (0 .. people-1)
            - "couples" : bool, whether couples are formed (0,1),(2,3),...
    games : int
        Number of independent games to simulate.
    seed : int
        Seed for the random number generator.
    
    Returns
    -------
    dict
        Dictionary with a single key "p" containing the estimated failure probability.
    """
    people = int(params["people"])
    couples = bool(params["couples"])
    rng = random.Random(seed)

    failures = 0

    for _ in range(games):
        remaining = list(range(people))          # names still in the hat
        for i in range(people):
            # Determine forbidden names for this person
            forbidden = {i}
            if couples:
                partner = i ^ 1                  # (0,1),(2,3),...
                if partner < people:
                    forbidden.add(partner)

            # Build list of allowed names
            allowed = [name for name in remaining if name not in forbidden]

            if not allowed:                     # no allowed name left -> failure
                failures += 1
                break

            # Choose uniformly at random from allowed names
            chosen = rng.choice(allowed)
            remaining.remove(chosen)            # name is taken out of the hat

    return {"p": failures / games}
qwen2.5-coder:7b · error · 4.8 s to write
  • {"people": 8, "couples": false}: timed out after 900s
  • {"people": 10, "couples": true}: timed out after 900s
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    people = params['people']
    couples = params['couples']
    rng = random.Random(seed)
    
    def draw_names(people, couples):
        names = list(range(people))
        if couples:
            for i in range(people // 2):
                names.remove(i)
                names.remove(i + people // 2)
        rng.shuffle(names)
        return names
    
    def simulate_game(people, couples):
        names = draw_names(people, couples)
        taken = set()
        for i in range(people):
            while True:
                name = names[rng.randint(0, len(names) - 1)]
                if name not in taken and name != i:
                    taken.add(name)
                    names.remove(name)
                    break
            if not names:
                return True
        return False
    
    fails = 0
    for _ in range(games):
        if simulate_game(people, couples):
            fails += 1
    
    probability = fails / games
    return {"p": probability}
qwen3.6:27b-mlx · agrees · 168.4 s to write
  • {"people": 8, "couples": false}: returned 0.09105, exact 0.0904946, z = +0.87
  • {"people": 10, "couples": true}: returned 0.246885, exact 0.24761, z = -0.75
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    rng = random.Random(seed)
    people = params["people"]
    couples = params["couples"]
    fail_count = 0
    
    for _ in range(games):
        hat = list(range(people))
        failed = False
        for i in range(people):
            if couples:
                partner = i ^ 1
                allowed = [x for x in hat if x != i and x != partner]
            else:
                allowed = [x for x in hat if x != i]
                
            if not allowed:
                failed = True
                break
                
            choice = allowed[rng.randrange(len(allowed))]
            hat.remove(choice)
            
        if failed:
            fail_count += 1
            
    return {"p": fail_count / games if games > 0 else 0.0}

Collecting: collector

The rules each model was given
A set has `items` different items, all equally common. Each pack contains `per_pack` items,
all different from each other within the pack, chosen uniformly at random among all such
combinations (packs are independent of each other). You buy packs one at a time until you
own every item at least once. Report the expected number of packs bought ("mean") and the
sample standard deviation of the number of packs ("sd").
gemma4:26b · agrees · 8.1 s to write
  • {"items": 50, "per_pack": 1}: returned 225.037, exact 224.96, z = +0.17
  • {"items": 100, "per_pack": 5}: returned 101.824, exact 102.051, z = -1.31
import random
import math

def simulate(params: dict, games: int, seed: int) -> dict:
    """
    Performs a Monte Carlo simulation to find the mean and standard deviation
    of the number of packs needed to collect all items.
    """
    items_total = params["items"]
    per_pack = params["per_pack"]
    
    rng = random.Random(seed)
    
    # Pre-calculate the range of items to avoid repeated range() calls
    item_pool = list(range(items_total))
    
    results = []
    
    for _ in range(games):
        collected = [False] * items_total
        num_collected = 0
        packs_bought = 0
        
        while num_collected < items_total:
            packs_bought += 1
            
            # random.sample is efficient for picking k unique elements from a population
            # It implements the selection without replacement logic required by the rules.
            pack = rng.sample(item_pool, per_pack)
            
            for item in pack:
                if not collected[item]:
                    collected[item] = True
                    num_collected += 1
        
        results.append(packs_bought)
    
    # Calculate Mean
    mean = sum(results) / games
    
    # Calculate Sample Standard Deviation
    # Formula: sqrt( sum((x - mean)^2) / (n - 1) )
    if games > 1:
        variance_sum = sum((x - mean) ** 2 for x in results)
        sd = math.sqrt(variance_sum / (games - 1))
    else:
        sd = 0.0
        
    return {"mean": mean, "sd": sd}
gpt-oss:20b · agrees · 109.4 s to write
  • {"items": 50, "per_pack": 1}: returned 224.838, exact 224.96, z = -0.28
  • {"items": 100, "per_pack": 5}: returned 101.999, exact 102.051, z = -0.30
import random
import math

def simulate(params: dict, games: int, seed: int) -> dict:
    """
    Monte‑Carlo simulation of buying packs until all items are collected.

    Parameters
    ----------
    params : dict
        Must contain:
            - 'items'   : total number of distinct items (int)
            - 'per_pack': number of distinct items per pack (int)
    games : int
        Number of independent simulation runs.
    seed : int
        Seed for the random number generator.

    Returns
    -------
    dict
        {"mean": <average number of packs>, "sd": <sample standard deviation>}
    """
    items = int(params["items"])
    per_pack = int(params["per_pack"])

    rng = random.Random(seed)
    results = []

    # Pre‑create the list of all item indices for sampling
    all_items = list(range(items))

    for _ in range(games):
        seen = set()
        packs = 0
        while len(seen) < items:
            packs += 1
            # Sample without replacement
            pack = rng.sample(all_items, per_pack)
            seen.update(pack)
        results.append(packs)

    # Compute mean and sample standard deviation
    n = games
    mean = sum(results) / n
    # sample variance (denominator n-1)
    var = sum((x - mean) ** 2 for x in results) / (n - 1) if n > 1 else 0.0
    sd = math.sqrt(var)

    return {"mean": mean, "sd": sd}
qwen2.5-coder:7b · agrees · 6.6 s to write
  • {"items": 50, "per_pack": 1}: returned 224.651, exact 224.96, z = -0.71
  • {"items": 100, "per_pack": 5}: returned 101.943, exact 102.051, z = -0.62
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    items = params['items']
    per_pack = params['per_pack']
    rng = random.Random(seed)
    
    def simulate_one_game():
        owned = set()
        packs_bought = 0
        while len(owned) < items:
            pack = set(rng.sample(range(items), per_pack))
            owned.update(pack)
            packs_bought += 1
        return packs_bought
    
    results = [simulate_one_game() for _ in range(games)]
    mean = sum(results) / games
    variance = sum((x - mean) ** 2 for x in results) / (games - 1)
    sd = variance ** 0.5
    
    return {"mean": mean, "sd": sd}
qwen3.6:27b-mlx · agrees · 132.9 s to write
  • {"items": 50, "per_pack": 1}: returned 224.838, exact 224.96, z = -0.28
  • {"items": 100, "per_pack": 5}: returned 101.999, exact 102.051, z = -0.30
import random

def simulate(params: dict, games: int, seed: int) -> dict:
    items = params["items"]
    per_pack = params["per_pack"]
    rng = random.Random(seed)
    
    target = (1 << items) - 1
    bits = [1 << i for i in range(items)]
    range_items = range(items)
    
    results = []
    for _ in range(games):
        collected = 0
        packs = 0
        while collected != target:
            packs += 1
            pack_mask = 0
            for idx in rng.sample(range_items, per_pack):
                pack_mask |= bits[idx]
            collected |= pack_mask
        results.append(packs)
        
    mean = sum(results) / games
    if games > 1:
        sd = (sum((x - mean) ** 2 for x in results) / (games - 1)) ** 0.5
    else:
        sd = 0.0
    return {"mean": mean, "sd": sd}