AutoGrad From Scrach

References

• https://en.wikipedia.org/wiki/Differentiable_function

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# Set the random seed for reproducibility
np.random.seed(1000)

Differentiability

• A function f(x) is really differentiable when it has a well defined derivative at a given point x = a, and it’s derivative f’(a) exists and is finite. \begin{align*} f'(a) &= \lim_{h \rightarrow 0} \frac{f(a + h) - f(a)}{h} \\\\ &\text{or} \\ f'(a) &= \lim_{h \rightarrow 0} \frac{f(a + h) - f(a - h)}{2h} \\\\ \end{align*}

• In simpler terms if function has a slope or rate of exchange than can be calculated (must not have any sharp edges)

• If a function is not conitnuous at a point, it is not differentiable there, but not all conitnuous functions are differenciable

• Examples -

  • Differentiable \(f(x) = x^2\)

  • Non differentiable \(f(x) = |x|\)

Derivative

def diff_func_1(x):
    """Square function

    .. math::
        f(x) = x^2
    """
    return x**2

def diff_func_2(x):
    """Quadratic function

    .. math::
        f(x) = x^2 - 2x + 1
    """
    return x**2 - (2 * x) + 1

def diff_func_3(x):
    """Sine function

    .. math::
        f(x) = sin(x)
    """
    return np.sin(x)

def non_diff_func_1(x):
    """Absolute value function

    .. math::
        f(x) = |x|
    """
    return np.abs(x)

def non_diff_func_2(x):
    """Step function

    .. math::
        f(x) = 1 if x > 0, 0 otherwise
    """
    return x * np.abs(x)

def derivative(func, x):
    h = 1e-5
    f_x_plus_h = func(x + h)
    f_x_minus_h = func(x - h)
    return (f_x_plus_h - f_x_minus_h) / (2.0 * h)

x = 1
h = 1e-5

print(derivative(diff_func_1, x))

2.000000000002

def plot_differentiation(func):
    delta = 0.1  # Smaller step size for smooth curves
    lim = 5
    x = np.arange(-lim, lim + delta, delta)
    f_x = func(x)
    df_x = derivative(func, x)

    fig, ax = plt.subplots(1, 1, figsize=(4, 3))

    # Plot function and derivative
    ax.plot(x, f_x, label='f(x)', color='blue', linewidth=2)
    ax.plot(x, df_x, label="f'(x)", color='orange', linestyle='--', linewidth=2)

    # Highlight axes
    ax.axhline(0, color='black', linewidth=0.8, linestyle='--')
    ax.axvline(0, color='black', linewidth=0.8, linestyle='--')

    # Titles and labels
    ax.set_title("Function and its Derivative", fontsize=14)
    ax.set_xlabel("x", fontsize=12)
    ax.set_ylabel("y", fontsize=12)

    critical_points = x[np.abs(df_x) < 1e-3]  # Approximate zeros of the derivative
    ax.scatter(critical_points, func(critical_points), color='red', label='Critical Points', zorder=5)

    # Grid and legend
    ax.grid(True, linestyle='--', alpha=0.6)
    ax.legend(fontsize=12)

    fig.show();

plot_differentiation(diff_func_1)

plot_differentiation(non_diff_func_1)