Calculating p-values with python.

Libraries and helper functions

from typing import Tuple
import math as m
def normal_probability_below(x: float, mu: float = 0, sigma: float = 1) -> float:
    return (1 + m.erf((x - mu) / m.sqrt(2) / sigma)) / 2
def normal_probability_above(lo: float, mu: float = 0, sigma: float = 1) -> float:
    return 1 - normal_probability_below(lo, mu, sigma)
def normal_approximation_to_binomial(n: int, p: float) -> Tuple[float, float]:
    mu = p * n
    sigma = m.sqrt(p * (1 - p) * n)

    return mu, sigma

Two-sided p-values

def two_sided_p_value(x: float, mu: float = 0, sigma: float = 1) -> float:
    """Return the probability of getting at least as extreme value as `x`, given
    that our values are from a normal distribution with `mu` mean and `sigma` std.
    """

    # If x is greater than the mean return everything above x
    if x >= mu:
        return 2 * normal_probability_above(x, mu, sigma)
    # If x is less than the mean than return everything below x
    else:
        return 2 * normal_probability_below(x, mu, sigma)

E.g. if our normal distribution has a mean of 500 and standard deviation of 15, the probabilty to get 530 or above is ~5.78%

mu_0, sigma_0 = normal_approximation_to_binomial (1000 , 0.5 )
mu_0, sigma_0
(500.0, 15.811388300841896)
two_sided_p_value(529.5, mu_0, sigma_0)
0.06207721579598835

As the p-value is greater than 5%, so we don't reject the null hypothesis.

However, if we look at values beyond 32 distance from the mean, however, the probability will be less than 5%.

two_sided_p_value(531.5, mu_0, sigma_0)
0.046345287837786575

Validating with simulation

import random

# Run 1000 simulations with 1000 binomial trials
extreme_value_count = 0
for _ in range(1000):
    num_heads = sum(1 if random.random() < 0.5 else 0 for _ in range(1000))

    # Count the 'extreme' values (i.e. beyond 30 distance from the mean)
    if num_heads >= 530 or num_heads <= 470:
        extreme_value_count += 1

extreme_value_count / 1000
# assert 610 < extreme_value_count < 630, f"{extreme_value_count}"
0.07

One-sided p-values

normal_probability_above(524.5, mu_0, sigma_0)
0.06062885772582072
normal_probability_above(526.5, mu_0, sigma_0)
0.04686839508859242