% Source: https://kexue.fm/archives/11626
\documentclass[11pt]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage{geometry}
\usepackage{hyperref}
\usepackage{graphicx}
\usepackage{amsmath,amssymb}
\usepackage{booktabs,longtable,array}
\usepackage{xcolor}
\usepackage{listings}
\usepackage{caption}
\usepackage{float}
\usepackage{url}

\geometry{a4paper, margin=1in}
\setlength{\parindent}{0pt}
\setlength{\parskip}{6pt}

\title{MoE Around the World: 7, A Minimalist Solution for Dynamic Activation}
\author{Su Jianlin}
\date{2026-02-23}

\begin{document}
\maketitle

\section*{Dynamic Activation}
In the previous article \href{https://kexue.fm/archives/11619}{\emph{MoE Around the World: 6, Promoting Equilibrium via Optimal Allocation}}, we achieved load balancing by solving the following optimal assignment problem:
\begin{equation}
\max_{x_{i,j}\in\{0,1\}} \sum_{i,j} x_{i,j}s_{i,j} \qquad\text{s.t.}\qquad \sum_j x_{i,j} = k,\quad \sum_i x_{i,j} = \frac{mk}{n}
\end{equation}
Here, $\sum_j x_{i,j} = k$ means each token activates exactly $k$ experts, and $\sum_i x_{i,j} = mk/n$ means each expert is activated exactly $mk/n$ times. However, upon closer inspection, we realize that the first constraint is not essential for training or inference. What we truly need is the second one: it implies that ``\emph{on average}, each token activates $k$ experts'' and ensures each expert's load is balanced---this alone suffices for MoE's goals. Thus, this paper considers a simpler problem:
\begin{equation}
\max_{x_{i,j}\in\{0,1\}} \sum_{i,j} x_{i,j}s_{i,j} \qquad\text{s.t.}\qquad \sum_i x_{i,j} = \frac{mk}{n}\label{eq:target-dyn}
\end{equation}

Assuming familiarity with the previous article, derivations here will be more concise; revisit the prior piece if any steps are unclear. As before, consider the relaxed version of objective \eqref{eq:target-dyn}:
\begin{equation}
\max_{x_{i,j}\in[0,1]} \sum_{i,j} x_{i,j}s_{i,j} \qquad\text{s.t.}\qquad \sum_i x_{i,j} = \frac{mk}{n}
\end{equation}
Then consider its equivalent $\max\text{-}\min$ form:
\begin{equation}
\max_{x_{i,j}\in[0,1]}\min_{\beta_j} \sum_{i,j} x_{i,j}s_{i,j}  - \sum_j \beta_j\left(\sum_i x_{i,j} - \frac{mk}{n}\right)
\end{equation}
Similarly, the $\max$ and $\min$ can be swapped, leading to:
\begin{equation}
\min_{\beta_j}\max_{x_{i,j}\in[0,1]} \sum_{i,j} x_{i,j}(s_{i,j} - \beta_j) + \frac{mk}{n} \sum_j \beta_j\label{eq:relax-min-max}
\end{equation}
The $\max$ step can be solved first, yielding:
\begin{equation}
\left\{\begin{aligned}
&\,x_{i,j}^* = 1, &\, s_{i,j} - \beta_j > 0 \\
&\,x_{i,j}^* = 0, &\, s_{i,j} - \beta_j < 0 \\
&\,x_{i,j}^* \in  [0,1], &\, s_{i,j} - \beta_j = 0
\end{aligned}\right.
\end{equation}
This tells us that any expert with $s_{i,j} - \beta_j > 0$ will be activated, and the number of activated experts is uncertain. This formally aligns with \href{https://kexue.fm/archives/10815}{\emph{MoE Around the World: 4, Invest More Where It's Harder}}, except that here we derive it from the more fundamental objective \eqref{eq:target-dyn} instead of designing it heuristically.

\section*{One-Step Solution}
Substituting $x_{i,j}^*$ back into \eqref{eq:relax-min-max} gives $x_{i,j}^*(s_{i,j} - \beta_j) = \max(0, s_{i,j} - \beta_j)$, so the optimization objective for $\beta_j$ simplifies to:
\begin{equation}
\min_{\beta_j} \sum_{i,j} \max(0, s_{i,j} - \beta_j) + \frac{mk}{n} \sum_j \beta_j\label{eq:beta-obj}
\end{equation}
Each $\beta_j$ term is independent, so this decomposes into $m$ independent sub-problems. We can thus temporarily omit the index $j$:
\begin{equation}
\min_{\beta} \frac{mk}{n}\beta + \sum_i \max(0, s_i - \beta)
\end{equation}
Sort all $s_i$ in descending order as $s_{\sigma_1}\geq s_{\sigma_2} \geq \cdots \geq s_{\sigma_m}$, and suppose we know $s_{\sigma_l}\geq\beta\geq s_{\sigma_{l+1}}$. Then we can eliminate $\max$:
\begin{equation}
\frac{mk}{n}\beta + \sum_{i=1}^l (s_{\sigma_i} - \beta) = \left\{\begin{aligned}
&\,\sum_{i=1}^{mk/n} s_{\sigma_i} + \sum_{i=mk/n+1}^l \underbrace{(s_{\sigma_i} - \beta)}_{\geq 0},&\, l \geq mk/n \\
&\,\sum_{i=1}^{mk/n} s_{\sigma_i} - \sum_{\hphantom{ab}i=l+1\hphantom{ab}}^{mk/n} \underbrace{(s_{\sigma_i} - \beta)}_{\leq 0},&\, l \leq mk/n \\
\end{aligned}\right.
\end{equation}
This shows that both $l > mk/n$ and $l < mk/n$ enlarge the objective, so the minimum occurs at $l=mk/n$, i.e., $\beta^*$ lies between the $(mk/n)$-th and $(mk/n+1)$-th largest elements of $s_i$. By convention, take the $(mk/n+1)$-th largest element. Restoring index $j$, $\beta_j^*$ is the $(mk/n+1)$-th largest element when $s_{i,j}$ is sorted in descending order, also known as the ``$1-k/n$ quantile.''

Note that the only variable now is $\boldsymbol{\beta}$, so there is no alternating iteration step---just compute the quantile once to get the optimal solution for perfect balance! We still call this ``\textbf{\textcolor{red}{Quantile Balancing (QB)}}''.

\section*{Practical Considerations}
For readers familiar with \href{https://kexue.fm/archives/11619}{\emph{MoE Around the World: 6, Promoting Equilibrium via Optimal Allocation}}, this paper presents a simplified version. However, simplicity does not imply triviality: it is actually more elegant and efficient than Top-$k$ MoE. First, it activates experts via the positivity of $\boldsymbol{s} - \boldsymbol{\beta}$, which is more efficient than selecting Top-$k$. Second, it obtains the load-balanced optimal solution in one step via quantile, again more efficient and elegant.

Of course, a one-step optimal solution may overfit the current batch of training data. To mitigate this, we consider an improvement: apply EMA (Exponential Moving Average) to blend the current optimal solution with the historical one (note the Top-$k$ version in the previous article could similarly use EMA). Crucially, we must avoid information leakage, so we first use the old $\boldsymbol{\beta}$ to activate experts, then update $\boldsymbol{\beta}$.

The algorithm flow is as follows (in the author's experiments, $\lambda=0.9$):
\[
\begin{array}{|l|}
\hline
\text{Quantile Balancing (QB) Dynamic Version} \\[4pt]
\hline
\text{Input: score matrix } \boldsymbol{s}\in\mathbb{R}^{m\times n}, \text{ previous } \boldsymbol{\beta}\in\mathbb{R}^n, \text{ decay rate } \lambda \\
\text{Output: assignment } \boldsymbol{x}\in\{0,1\}^{m\times n}, \text{ updated } \boldsymbol{\beta}\in\mathbb{R}^n \\[4pt]
\hline
\begin{array}{ll}
1: & x_{i,j}=1 \text{ if } s_{i,j} - \beta_j > 0 \text{ else } 0 \\
2: & \boldsymbol{\beta} \leftarrow \lambda\boldsymbol{\beta} + (1-\lambda)\mathop{\text{des\_sort}}(\boldsymbol{s}, \text{axis=0})_{[mk/n:mk/n+1]} \\
3: & \text{Output } \boldsymbol{x},\boldsymbol{\beta}
\end{array} \\
\hline
\end{array}
\]

In practice, computing the $1-k/n$ quantile of $\boldsymbol{s}$ can be costly---it requires finding the $(mk/n+1)$-th largest element among $m$ elements, where $m$ equals ``global batch size × sequence length.'' Due to parallelism and gradient accumulation constraints, an exact implementation is often impractical. Thus, we still consider a compromise: split samples into the largest acceptable mini-batches, compute a $\boldsymbol{\beta}$ for each, and average them for the final result.

\section*{Expert Choice}
Interestingly, this dynamic QB has a very intuitive interpretation: ``Expert Choice.'' 
\begin{quote}
You want each expert to be activated exactly $mk/n$ times? Then let each expert choose its favorite $mk/n$ tokens---i.e., select the top-$mk/n$ tokens along $\text{axis=0}$---which is exactly finding the $mk/n$ largest elements.
\end{quote}
However, naive Expert Choice suffers from cross-sample and cross-sequence issues, violating causality and consistency between training and inference. Here, we cleverly convert it into a bias form: by setting $\boldsymbol{\beta}_j$ as the $1-k/n$ quantile for each expert, activation via the sign of $\boldsymbol{s}-\boldsymbol{\beta}$ implements Expert Choice. Then, by delaying the update of $\boldsymbol{\beta}$ until after activation, we avoid information leakage.

\section*{Initialization Strategy}
To prevent information leakage, during training we must first use the old $\boldsymbol{\beta}$ to determine which experts each token activates, then update $\boldsymbol{\beta}$. This makes early training steps sensitive to $\boldsymbol{\beta}$ initialization. For example, if $\boldsymbol{s}$ is entirely positive but $\boldsymbol{\beta}$ is initialized to zero, every token will activate all experts in the first training step, causing computational explosion.

In \href{https://kexue.fm/archives/10815}{\emph{MoE Around the World: 4, Invest More Where It's Harder}}, we provided a simulation script to estimate suitable initialization. Now, since we know the optimal $\boldsymbol{\beta}$ is the ``$1-k/n$ quantile,'' we can make reasonable assumptions about initial scores and derive an analytical initialization. Specifically, assume Router logits follow $\mathcal{N}(0,\sigma^2)$; then its $1-k/n$ quantile is:
\begin{equation}
\sigma \cdot \Phi^{-1}\left(1 - \frac{k}{n}\right)
\end{equation}
This is the optimal $\boldsymbol{\beta}$ for such logits, and we use it to initialize $\boldsymbol{\beta}$, where $\Phi^{-1}$ is the \href{https://en.wikipedia.org/wiki/Cumulative_distribution_function#Inverse_distribution_function_(quantile_function)}{quantile function} of the standard normal distribution (also known as the inverse CDF).

Additionally, if the Router uses an elementwise monotonic activation function (e.g., Sigmoid), it preserves order, so we can simply apply the activation to the above expression. With Softmax (which exponentiates then normalizes), it's slightly more complex: assuming the normalization denominator is approximately constant, it behaves similarly---just exponentiate the expression and divide by an estimated denominator via quantile sampling:
\begin{equation}
\frac{\exp\left[\sigma \cdot \Phi^{-1}\left(1 - \frac{k}{n}\right)\right]}{\sum\limits_{i=1}^n \exp\left[\sigma \cdot \Phi^{-1}\left(1 - \frac{i}{n+1}\right)\right]}
\end{equation}

As for estimating $\sigma$, it’s straightforward. Assume the Router’s input dimension is $d$, and RMS Norm is applied. If the Router’s weight matrix is initialized with $\mathcal{N}(0,\tilde{\sigma}^2)$, then router logits approximate a normal distribution with mean 0 and variance $\tilde{\sigma}^2 d$, i.e., $\sigma\approx \tilde{\sigma} \sqrt{d}$---these are classic initialization results \cite{https://kexue.fm/archives/8620}.

\section*{Demonstration Code}
We include a demonstration code snippet for experimentation:
\begin{verbatim}
import numpy as np

def quantile_bias(s, k):
    """Dynamic version of QB: one quantile step gives optimal bias
    Principle: https://kexue.fm/archives/11626
    """
    m, n = s.shape
    beta = np.quantile(s, 1 - k / n, axis=0)
    return beta

def max_min_avg_vio(s):
    """Compute max_vio, min_vio, avg_vio
    where max_vio >= 0, avg_vio >= 0, -1 <= min_vio <= 0; all closer to 0 means better balance
    """
    m, n = s.shape
    f = (s > 0).mean(0)
    f = f / f.sum() * n - 1
    return f.max(), f.min(), np.abs(f).mean()

def avg_std_active(s):
    """Compute mean and std of expert activation counts
    mean closer to k is better, std closer to 0 is better
    """
    m, n = s.shape
    f = (s > 0).mean(0) * n
    return f.mean(), f.std()

m, n, k = 100000, 256, 8
s = np.random.rand(m, n) + np.random.rand(n)  # Simulate non-uniform scores

b = quantile_bias(s, k)
max_min_avg_vio(s - b)  # Should be all zero if no error
avg_std_active(s - b)  # Should be roughly (k, 0)


from scipy.stats import norm

sigma = 1
s = np.random.randn(m, n) * sigma  # Simulate initialization
avg_std_active(s)  # Roughly (128, 0.4), much higher than budget k
b0 = np.zeros(n) + norm.ppf(1 - k / n) * sigma  # norm.ppf is the standard normal quantile function
avg_std_active(s - b0)  # Roughly (8, 0.1), close to budget k

s2 = 1 / (1 + np.exp(-s))  # Sigmoid activation
b0_2 = 1 / (1 + np.exp(-b0))  # Sigmoid activation
avg_std_active(s2 - b0_2)  # Still roughly (8, 0.1), close to budget k

s3 = np.exp(s) / np.exp(s).sum(axis=1, keepdims=True)  # Softmax activation
q = (np.arange(n) + 1) / (n + 1)  # Uniformly spaced points (quantiles)
b0_3 = np.exp(b0) / np.exp(sigma * norm.ppf(q)).sum() # Exponentiate and divide by estimated denominator
avg_std_active(s3 - b0_3)  # Roughly (7.5, 0.1), close to budget k
\end{verbatim}

\section*{Gradient Descent}
Finally, for readers who prefer not to use quantiles, gradient descent remains an option. Denote the loss function from objective \eqref{eq:beta-obj} as $\ell$:
\begin{equation}
\min_{\beta_j} \underbrace{\sum_{i,j} \max(0, s_{i,j} - \beta_j) + \frac{mk}{n} \sum_j \beta_j}_{\text{denoted } \ell}
\end{equation}
It is differentiable, with gradient:
\begin{equation}
\frac{\partial\ell}{\partial\beta_j} = \frac{mk}{n} - \sum_{i=1}^m \chi(s_{i,j} - \beta_j > 0)
\end{equation}
where $\chi$ is the indicator function, $\chi(\text{True})=1,\chi(\text{False})=0$. The gradient computation is cheaper than a global quantile, suitable for cost-sensitive readers. Given the gradient, perform gradient descent---using SignSGD to align with Loss-Free:
\begin{equation}
\beta_j \leftarrow \beta_j - \gamma\mathop{\text{sign}}\left(\frac{\partial\ell}{\partial\beta_j}\right)
\end{equation}
This can replace the quantile update for $\boldsymbol{\beta}$, and since it's a long-term iterative algorithm, EMA can be removed. Indeed, this is exactly formula (9) in \href{https://kexue.fm/archives/10815}{\emph{MoE Around the World: 4, Invest More Where It's Harder}}, re-derived from the perspective of ``optimal assignment + dual objective + gradient descent.''

\section*{Summary}
This article extends the previous Quantile Balancing (QB) method by removing the ``each token activates exactly $k$ experts'' constraint from the optimal assignment problem, significantly simplifying the solution: only one quantile step is needed to achieve load balance. Each token decides which experts to activate simply by checking the sign of bias-adjusted scores, eliminating the cost of Top-$k$ sorting.

\end{document}
