New script demonstrates the implementation of the Sigmoid, tanh function, Matrix determinant and Matrix trace calculation.

linear_algebra/determinant.py

@@ -0,0 +1,191 @@
+"""
+Matrix determinant calculation using various methods.
+
+The determinant is a scalar value that characterizes a square matrix.
+It provides important information about the matrix, such as whether it's invertible.
+
+Reference: https://en.wikipedia.org/wiki/Determinant
+"""
+
+import numpy as np
+from numpy import float64
+from numpy.typing import NDArray
+
+
+def determinant_recursive(matrix: NDArray[float64]) -> float:
+    """
+    Calculate the determinant of a square matrix
+    using recursive cofactor expansion.
+    This method is suitable for
+    small matrices but becomes inefficient for large matrices.
+    Parameters:
+        matrix (NDArray[float64]): A square matrix
+    Returns:
+        float: The determinant of the matrix
+    Raises:
+        ValueError: If the matrix is not square
+    Examples:
+    >>> import numpy as np
+    >>> matrix = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=float)
+    >>> determinant_recursive(matrix)
+    -2.0
+
+    >>> matrix = np.array([[5.0]], dtype=float)
+    >>> determinant_recursive(matrix)
+    5.0
+    """
+    if matrix.shape[0] != matrix.shape[1]:
+        raise ValueError("Matrix must be square")
+
+    n = matrix.shape[0]
+
+    # Base cases
+    if n == 1:
+        return float(matrix[0, 0])
+
+    if n == 2:
+        return float(matrix[0, 0] * matrix[1, 1] - matrix[0, 1] * matrix[1, 0])
+
+    # Recursive case: cofactor expansion along the first row
+    det = 0.0
+    for col in range(n):
+        # Create submatrix by removing row 0 and column col
+        submatrix = np.delete(np.delete(matrix, 0, axis=0), col, axis=1)
+
+        # Calculate cofactor
+        cofactor = ((-1) ** col) * matrix[0, col] * determinant_recursive(submatrix)
+        det += cofactor
+
+    return det
+
+
+def determinant_lu(matrix: NDArray[float64]) -> float:
+    """
+    Calculate the determinant using LU decomposition.
+    This method is more efficient for larger matrices
+    than recursive expansion.
+    Parameters:
+        matrix (NDArray[float64]): A square matrix
+    Returns:
+        float: The determinant of the matrix
+    Raises:
+        ValueError: If the matrix is not square
+    """
+    if matrix.shape[0] != matrix.shape[1]:
+        raise ValueError("Matrix must be square")
+
+    n = matrix.shape[0]
+
+    # Create a copy to avoid modifying the original matrix
+    copy = matrix.astype(float64, copy=True)
+
+    # Keep track of row swaps for sign adjustment
+    swap_count = 0
+
+    # Forward elimination to get upper triangular matrix
+    for i in range(n):
+        # Find pivot
+        max_row = i
+        for k in range(i + 1, n):
+            if abs(copy[k, i]) > abs(copy[max_row, i]):
+                max_row = k
+
+        # Swap rows if needed
+        if max_row != i:
+            copy[[i, max_row]] = copy[[max_row, i]]
+            swap_count += 1
+
+        # Check for singular matrix
+        if abs(copy[i, i]) < 1e-14:
+            return 0.0
+
+        # Eliminate below pivot
+        for k in range(i + 1, n):
+            factor = copy[k, i] / copy[i, i]
+            for j in range(i, n):
+                copy[k, j] -= factor * copy[i, j]
+
+    # Calculate determinant as product of diagonal elements
+    det = 1.0
+    for i in range(n):
+        det *= copy[i, i]
+
+    # Adjust sign based on number of row swaps
+    if swap_count % 2 == 1:
+        det = -det
+
+    return det
+
+
+def determinant(matrix: NDArray[float64]) -> float:
+    """
+    Calculate the determinant of a square matrix using
+    the most appropriate method.
+    Uses recursive expansion for small matrices (≤3x3)
+    and LU decomposition for larger ones.
+    Parameters:
+        matrix (NDArray[float64]): A square matrix
+    Returns:
+        float: The determinant of the matrix
+    Examples:
+    >>> import numpy as np
+    >>> matrix = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=float)
+    >>> determinant(matrix)
+    -2.0
+    """
+    if matrix.shape[0] != matrix.shape[1]:
+        raise ValueError("Matrix must be square")
+
+    n = matrix.shape[0]
+
+    # Use recursive method for small matrices, LU decomposition for larger ones
+    if n <= 3:
+        return determinant_recursive(matrix)
+    else:
+        return determinant_lu(matrix)
+
+
+def test_determinant() -> None:
+    """
+    Test function for matrix determinant calculation.
+
+    >>> test_determinant()  # self running tests
+    """
+    # Test 1: 2x2 matrix
+    matrix_2x2 = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=float)
+    det_2x2 = determinant(matrix_2x2)
+    assert abs(det_2x2 - (-2.0)) < 1e-10, "2x2 determinant calculation failed"
+
+    # Test 2: 3x3 matrix
+    matrix_3x3 = np.array(
+        [[2.0, -3.0, 1.0], [2.0, 0.0, -1.0], [1.0, 4.0, 5.0]], dtype=float
+    )
+    det_3x3 = determinant(matrix_3x3)
+    assert abs(det_3x3 - 49.0) < 1e-10, "3x3 determinant calculation failed"
+
+    # Test 3: Singular matrix
+    singular_matrix = np.array([[1.0, 2.0], [2.0, 4.0]], dtype=float)
+    det_singular = determinant(singular_matrix)
+    assert abs(det_singular) < 1e-10, "Singular matrix should have zero determinant"
+
+    # Test 4: Identity matrix
+    identity_3x3 = np.eye(3, dtype=float)
+    det_identity = determinant(identity_3x3)
+    assert abs(det_identity - 1.0) < 1e-10, "Identity matrix should have determinant 1"
+
+    # Test 5: Compare recursive and LU methods
+    test_matrix = np.array(
+        [[1.0, 2.0, 3.0], [0.0, 1.0, 4.0], [5.0, 6.0, 0.0]], dtype=float
+    )
+    det_recursive = determinant_recursive(test_matrix)
+    det_lu = determinant_lu(test_matrix)
+    assert abs(det_recursive - det_lu) < 1e-10, (
+        "Recursive and LU methods should give same result"
+    )
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()
+    test_determinant()

linear_algebra/matrix_trace.py

@@ -0,0 +1,143 @@
+"""
+Matrix trace calculation.
+
+The trace of a square matrix is the sum of the elements on the main diagonal.
+It's an important linear algebra operation with many applications.
+
+Reference: https://en.wikipedia.org/wiki/Trace_(linear_algebra)
+"""
+
+import numpy as np
+from numpy import float64
+from numpy.typing import NDArray
+
+
+def trace(matrix: NDArray[float64]) -> float:
+    """
+    Calculate the trace of a square matrix.
+
+    The trace is the sum of the diagonal elements of a square matrix.
+
+    Parameters:
+        matrix (NDArray[float64]): A square matrix
+
+    Returns:
+        float: The trace of the matrix
+
+    Raises:
+        ValueError: If the matrix is not square
+
+    Examples:
+    >>> import numpy as np
+    >>> matrix = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=float)
+    >>> trace(matrix)
+    5.0
+
+    >>> matrix = np.array(
+    ...     [[2.0, -1.0, 3.0], [4.0, 5.0, -2.0], [1.0, 0.0, 7.0]], dtype=float
+    ... )
+    >>> trace(matrix)
+    14.0
+
+    >>> matrix = np.array([[5.0]], dtype=float)
+    >>> trace(matrix)
+    5.0
+    """
+    if matrix.shape[0] != matrix.shape[1]:
+        raise ValueError("Matrix must be square")
+
+    return float(np.sum(np.diag(matrix)))
+
+
+def trace_properties_demo(matrix: NDArray[float64]) -> dict:
+    """
+    Demonstrate various properties of the trace operation.
+
+    Parameters:
+        matrix (NDArray[float64]): A square matrix
+
+    Returns:
+        dict: Dictionary containing trace properties and calculations
+    """
+    if matrix.shape[0] != matrix.shape[1]:
+        raise ValueError("Matrix must be square")
+
+    n = matrix.shape[0]
+
+    # Calculate trace
+    tr = trace(matrix)
+
+    # Calculate transpose trace (should be equal to original)
+    tr_transpose = trace(matrix.T)
+
+    # Calculate trace of scalar multiple
+    scalar = 2.0
+    tr_scalar = trace(scalar * matrix)
+
+    # Create identity matrix for comparison
+    identity = np.eye(n, dtype=float64)
+    tr_identity = trace(identity)
+
+    return {
+        "original_trace": tr,
+        "transpose_trace": tr_transpose,
+        "scalar_multiple_trace": tr_scalar,
+        "scalar_factor": scalar,
+        "identity_trace": tr_identity,
+        "trace_equals_transpose": abs(tr - tr_transpose) < 1e-10,
+        "scalar_property_check": abs(tr_scalar - scalar * tr) < 1e-10,
+    }
+
+
+def test_trace() -> None:
+    """
+    Test function for matrix trace calculation.
+
+    >>> test_trace()  # self running tests
+    """
+    # Test 1: 2x2 matrix
+    matrix_2x2 = np.array([[1.0, 2.0], [3.0, 4.0]], dtype=float)
+    tr_2x2 = trace(matrix_2x2)
+    assert abs(tr_2x2 - 5.0) < 1e-10, "2x2 trace calculation failed"
+
+    # Test 2: 3x3 matrix
+    matrix_3x3 = np.array(
+        [[2.0, -1.0, 3.0], [4.0, 5.0, -2.0], [1.0, 0.0, 7.0]], dtype=float
+    )
+    tr_3x3 = trace(matrix_3x3)
+    assert abs(tr_3x3 - 14.0) < 1e-10, "3x3 trace calculation failed"
+
+    # Test 3: Identity matrix
+    identity_4x4 = np.eye(4, dtype=float)
+    tr_identity = trace(identity_4x4)
+    assert abs(tr_identity - 4.0) < 1e-10, (
+        "Identity matrix trace should equal dimension"
+    )
+
+    # Test 4: Zero matrix
+    zero_matrix = np.zeros((3, 3), dtype=float)
+    tr_zero = trace(zero_matrix)
+    assert abs(tr_zero) < 1e-10, "Zero matrix should have zero trace"
+
+    # Test 5: Trace properties
+    test_matrix = np.array(
+        [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], dtype=float
+    )
+    properties = trace_properties_demo(test_matrix)
+    assert properties["trace_equals_transpose"], "Trace should equal transpose trace"
+    assert properties["scalar_property_check"], "Scalar multiplication property failed"
+
+    # Test 6: Diagonal matrix
+    diagonal_matrix = np.diag([1.0, 2.0, 3.0, 4.0])
+    tr_diagonal = trace(diagonal_matrix)
+    expected = 1.0 + 2.0 + 3.0 + 4.0
+    assert abs(tr_diagonal - expected) < 1e-10, (
+        "Diagonal matrix trace should equal sum of diagonal elements"
+    )
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()
+    test_trace()

neural_network/activation_functions/sigmoid.py

@@ -0,0 +1,40 @@
+"""
+This script demonstrates the implementation of the Sigmoid function.
+
+The sigmoid function is a logistic function, which describes growth as being initially
+exponential, but then slowing down and barely growing at all when a limit is reached.
+It's commonly used as an activation function in neural networks.
+
+For more detailed information, you can refer to the following link:
+https://en.wikipedia.org/wiki/Sigmoid_function
+"""
+
+import numpy as np
+
+
+def sigmoid(vector: np.ndarray) -> np.ndarray:
+    """
+    Implements the sigmoid activation function.
+
+    Parameters:
+        vector (np.ndarray): A vector that consists of numeric values
+
+    Returns:
+        np.ndarray: Input vector after applying sigmoid activation function
+
+    Formula: f(x) = 1 / (1 + e^(-x))
+
+    Examples:
+    >>> sigmoid(np.array([-1.0, 0.0, 1.0, 2.0]))
+    array([0.26894142, 0.5       , 0.73105858, 0.88079708])
+
+    >>> sigmoid(np.array([-5.0, -2.5, 2.5, 5.0]))
+    array([0.00669285, 0.07585818, 0.92414182, 0.99330715])
+    """
+    return 1 / (1 + np.exp(-vector))
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()