AI & compute

The numeric core underneath serez-ai: tensors, the autodiff tape that trains them, and the GPU escape hatch when the CPU is not enough.

Tensor

Multi-dimensional arrays with automatic differentiation support. All operations are tracked when a tape is active.

Creation

Tensor.zeros([rows, cols])
Tensor.ones([rows, cols])
Tensor.eye(n)                            // identity matrix
Tensor.from([[1.0, 2.0], [3.0, 4.0]])    // from a (nested) array
Random.normalTensor([rows, cols], 0.0, 0.1)  // gaussian-filled

Shape manipulation

t.shape()        // → [rows, cols, ...]
t.ndim()         // → number of dimensions
t.size()         // → total number of elements
t.reshape([2,3]) // reshape (same total elements)
t.flatten()      // → 1D tensor
t.transpose()    // 2D transpose
t.permute([0,2,1])   // N-D generalized transpose
t.unsqueeze(dim)     // insert size-1 dim at position
t.squeeze()          // remove all size-1 dims
t.squeeze(dim)       // remove specific size-1 dim

Math operations

// Element-wise (all tracked)
t.add(other)   t.sub(other)   t.mul(other)   t.div(other)
t.neg()        t.abs()        t.sqrt()       t.exp()
t.log()        t.pow(exp)     t.scale(s)
t.sign()       t.reciprocal()
t.sin()        t.cos()
t.round()      t.floor()      t.ceil()
t.clamp(min, max)
t.maximum(other)   t.minimum(other)

// Matrix operations (tracked)
t.dot(other)        // 1D dot product
t.matmul(other)     // 2D matrix multiply
t.outer(other)      // outer product
t.bmm(other)        // batch matmul [B,N,M] @ [B,M,K] → [B,N,K]

// Reductions
t.sum()   t.mean()   t.max()   t.min()
t.sumAxis(axis)     t.meanAxis(axis)
t.reduceSum(axis)   t.reduceMean(axis)   t.reduceMax(axis)
t.variance()        t.std()
t.cumsum()          t.norm(order)

// N-D broadcasting
t.broadcastTo([shape])             // expand to target shape
t.broadcastAdd(bias)               // 2D + 1D (tracked)
t.broadcastAddNd(other)            // N-D broadcast add
t.broadcastMulNd(other)            // N-D broadcast multiply

Activation functions (all tracked)

t.relu()
t.sigmoid()
t.tanh()
t.softmax()
t.gelu()
t.leaky_relu(alpha)
t.elu(alpha)           // ELU (alpha default 1.0)
t.swish()              // swish(x) = x * sigmoid(x)
t.silu()               // alias for swish
t.mish()               // mish(x) = x * tanh(softplus(x))
t.softplus()           // log(1 + exp(x))
t.hardsigmoid()        // clamp((x+3)/6, 0, 1)
t.hardswish()          // x * hardsigmoid(x)

Convolution & pooling (all tracked)

// Input shape: [N, H, W, C_in]
// Weights shape: [C_in * kernel^2, C_out]
t.conv2d(weights, bias, kernel, stride)
t.max_pool2d(kernel, stride)
t.avg_pool2d(kernel, stride)   // average pooling

Recurrent layers (all tracked)

// LSTM — input [seq_len, input_size], returns [1, hidden_size]
t.lstm(wx, wh, b, h0, c0)

// GRU — input [seq_len, input_size], returns [1, hidden_size]
t.gru(wx, wh, b, h0)

// Multi-head attention — input [seq_len, d_model]
t.mha(wq, wk, wv, wo, n_heads)

// Layer normalization — tracked
t.layer_norm(gamma, beta, eps)

Utilities

t.toArray()          // convert to serez array
t.toString()         // human-readable string
t.get(i, j)          // element access
t.set(i, j, val)     // element mutation
t.slice(start, end)  // flat slice
t.concat(other, axis)
t.argmax()   t.argmin()
t.stopGrad() // alias: t.detach() — detach from tape

Autodiff

Reverse-mode automatic differentiation tape. Record operations, run backward, retrieve gradients. All tensor operations performed while the tape is active are tracked automatically.

Tape control

Autodiff.tape()            // start recording
Autodiff.backward(loss)    // backpropagate from scalar tensor
Autodiff.gradient(tensor)  // retrieve gradient tensor
Autodiff.clear()           // clear tape and gradients
Autodiff.isRecording()     // → bool

Weight initialization

// Xavier (Glorot) — good for tanh / sigmoid
let w = Autodiff.xavierUniform([fan_in, fan_out])
let w = Autodiff.xavierNormal([fan_in, fan_out])

// He — good for ReLU networks
let w = Autodiff.heUniform([fan_in, fan_out])
let w = Autodiff.heNormal([fan_in, fan_out])

Optimizers

All optimizer steps are pure functions — they return updated parameters without modifying the tape.

// Adam — returns [new_param, new_m, new_v]
let result = Autodiff.adamStep(param, grad, m, v, step, lr)
let result = Autodiff.adamStep(param, grad, m, v, step, lr, beta1, beta2, eps)

// AdamW — Adam with decoupled weight decay
let result = Autodiff.adamwStep(param, grad, m, v, step, lr, wd)

// SGD with momentum — returns [new_param, new_velocity]
let result = Autodiff.sgdStep(param, grad, velocity, lr)
let result = Autodiff.sgdStep(param, grad, velocity, lr, momentum, weight_decay)

// RMSprop — returns [new_param, new_sq_avg]
let result = Autodiff.rmspropStep(param, grad, sq_avg, lr)
let result = Autodiff.rmspropStep(param, grad, sq_avg, lr, alpha, eps)

// Usage pattern:
let w = Autodiff.heNormal([128, 64])
let m = Tensor.zeros([128, 64])
let v = Tensor.zeros([128, 64])
let step = 0

// Training loop:
step++
Autodiff.tape()
let loss = Autodiff.mseLoss(w.matmul(x), target)
Autodiff.backward(loss)
let grad = Autodiff.gradient(w)
let res = Autodiff.adamStep(w, grad, m, v, step, 0.001)
w = res[0]; m = res[1]; v = res[2]

Loss functions

// All loss functions are tracked on the tape
let loss = Autodiff.mseLoss(pred, target)          // Mean Squared Error
let loss = Autodiff.maeLoss(pred, target)          // Mean Absolute Error
let loss = Autodiff.bceLoss(pred, target)          // Binary Cross-Entropy (probs in [0,1])
let loss = Autodiff.crossEntropyLoss(logits, idx)  // Cross-Entropy (raw logits + class indices)

Layers

// BatchNorm — normalizes [N, C] tensor per feature
let out = Autodiff.batchNorm(x, gamma, beta, training)
let out = Autodiff.batchNorm(x, gamma, beta, training, eps)

// Dropout — zeros random activations during training
let out = Autodiff.dropout(x, p)               // p = drop probability
let out = Autodiff.dropout(x, p, training)     // training=false → no-op

// Embedding — lookup rows from weight matrix
// indices: Array of int or integer Tensor
// weight:  [vocab_size, emb_dim] Tensor
// returns: [seq_len, emb_dim] Tensor
let out = Autodiff.embedding(indices, weight)

Gradient utilities

// Clip a single gradient tensor by norm
let clipped = Autodiff.clipGrad(grad, max_norm)

// Clip an array of gradients by global norm
let clipped = Autodiff.clipGradNorm([g1, g2, g3], max_norm)

// Detach a tensor from the tape (stop gradient flow)
let detached = Autodiff.stopGrad(tensor)
let detached = Autodiff.detach(tensor)

Weight persistence

// Save an array of tensors to a .szw binary file
Autodiff.saveWeights("model.szw", [w1, b1, w2, b2])

// Load tensors back — returns Array in same order
let weights = Autodiff.loadWeights("model.szw")
let w1 = weights[0]
let b1 = weights[1]

GPU

CPU-backed compute buffers with a GPU-shaped API. Buffers are flat decimal arrays; the create / upload / dispatch / readback / free pattern mirrors real GPU compute so a future backend can swap in actual GPU calls. Buffers are not garbage-collected — free them with GPU.freeBuffer when done. No permission declaration is required.

// Upload data, run element-wise + reduction, read back
let src     = GPU.createBufferFromArray([1.0, 2.0, 3.0, 4.0])  // → buffer id
let doubled = GPU.map(src, x => x * 2.0)            // element-wise → new buffer
let sum     = GPU.reduce(src, (acc, x) => acc + x, 0.0)   // → 10.0
let product = GPU.reduce(src, (acc, x) => acc * x, 1.0)   // → 24.0

// Linear algebra
let d = GPU.dot(src, doubled)        // dot product → decimal
let r = GPU.axpy(2.0, src, doubled)  // 2*src + doubled → new buffer

// Matrix multiply: [2×2] @ [2×2]
let I = GPU.createBufferFromArray([1.0, 0.0, 0.0, 1.0])
let M = GPU.createBufferFromArray([5.0, 6.0, 7.0, 8.0])
let C = GPU.matmul(I, 2, 2, M, 2, 2)
out GPU.readBuffer(C)   // → [5.0, 6.0, 7.0, 8.0]

// Always free buffers you created
GPU.freeBuffer(src)
GPU.freeBuffer(doubled)