keyATM with Collapsed Variational Bayes

Before you start

keyATMvb is an experimental function. Codes and derivations may contain errors. Please leave feedback in GitHub Issues.

Usage

library(keyATM)
data(keyATM_data_bills)
bills_dfm <- keyATM_data_bills$doc_dfm
bills_keywords <- keyATM_data_bills$keywords
keyATM_docs <- keyATM_read(bills_dfm)

out <- keyATMvb(
  docs = keyATM_docs,
  no_keyword_topics = 3,
  keywords = bills_keywords,
  model = "base",
  options = list(seed = 250),
  vb_options = list(convtol = 1e-4, init = "mcmc")
)

Derivation

Evidence Lower Bound

$$\begin{align} & \log p(\mathbf{w}\mid \boldsymbol{\alpha}, \boldsymbol{\beta}, \boldsymbol{\gamma}, \tilde{\boldsymbol{\beta}}) \\ &= \log \sum_{z,s} p(\mathbf{w}, \mathbf{z}, \mathbf{s}\mid \boldsymbol{\alpha}, \boldsymbol{\beta}, \tilde{\boldsymbol{\beta}}, \boldsymbol{\gamma}) \\ &= \log \sum q(\mathbf{z}, \mathbf{s}) \frac{p(\mathbf{w}, \mathbf{z}, \mathbf{s}\mid \boldsymbol{\alpha}, \boldsymbol{\beta}, \tilde{\boldsymbol{\beta}}, \boldsymbol{\gamma})}{q(\mathbf{z}, \mathbf{s})}\\ &\geq \sum q(\mathbf{z}, \mathbf{s}) \log \frac{p(\mathbf{w}, \mathbf{z}, \mathbf{s}\mid \boldsymbol{\alpha}, \boldsymbol{\beta}, \tilde{\boldsymbol{\beta}}, \boldsymbol{\gamma})}{q(\mathbf{z}, \mathbf{s})}\\ &= \mathbb{E}_q[\log p(\mathbf{w}, \mathbf{z}, \mathbf{s}\mid \boldsymbol{\alpha}, \boldsymbol{\beta}, \boldsymbol{\gamma})] - \mathbb{E}_q[\log q(\mathbf{z})] - \mathbb{E}_q[q(\mathbf{s})] \label{eq:ELBO} \end{align}$$ where we use Jensen’s inequality and factorization assumption.

Update parameters

Update $q(\mathbf{z})$

Extract terms related $q(z_{di})$ from ELBO. $$\begin{align} \mathcal{L}[q(z_{di})] &= \sum_{\mathbf{z}} q(z_{di}) q(\mathbf{z}^{-di}) q(s_{di}) q(\mathbf{s}^{-di}) \log \frac{p(w_{di}, z_{di}, s_{di}\mid \mathbf{w}^{-di}, \mathbf{z}^{-di}, \mathbf{s}^{-di}, \boldsymbol{\alpha}, \boldsymbol{\beta}, \boldsymbol{\gamma})}{q(z_{di})} \end{align}$$

We combine the results of Variational Bayes and Collapsed Gibbs Sampling. From Variational Bayes, $$\begin{align} &q(z_{di} = k) \propto \exp \bigg( \int q(s_{di} = 1) q(\tilde{\phi}_{kv}) \log \tilde{\phi}_{kv}\ d\tilde{\phi}_{kv} + \int q(s_{di} = 0) q({\phi}_{kv}) \log {\phi}_{kv}\ d{\phi}_{kv} \bigg) \\ &\quad \times \exp \bigg( \int q(\theta_{dk}) \log \theta_{dk}\ d\theta_{dk} \bigg) \times \exp \bigg( q(s_{di} = 1) \int q(\pi_k)\log \pi_k\ d\pi_k \bigg) \times \exp \bigg( \int q(\pi_k)\log \pi_k\ d\pi_k \bigg) \\ &= \exp \bigg( q(s_{di} = 1) \int q(\tilde{\phi}_{kv}) \log \tilde{\phi}_{kv}\ d\tilde{\phi}_{kv} + q(s_{di} = 1) \int q(\pi_k)\log \pi_k\ d\pi_k \bigg) \\ &\quad \times \exp \bigg( q(s_{di} = 0) \int q({\phi}_{kv}) \log {\phi}_{kv}\ d{\phi}_{kv} + q(s_{di} = 0) \int q(\pi_k)\log \pi_k\ d\pi_k \bigg) \times \exp \bigg( \int q(\theta_{dk}) \log \theta_{dk}\ d\theta_{dk} \bigg) \end{align}$$ % Results of Collapsed Gibbs Sampling show, $$\begin{align} &\quad \Pr(z_{di}=k \mid \mathbf{z}^{-di}, \mathbf{w}, \mathbf{s}, \boldsymbol{\alpha}, \boldsymbol{\beta}, \tilde{\boldsymbol{\beta}}, \boldsymbol{\gamma}) \propto \begin{cases} % \frac{\displaystyle \beta_v + n_{k v}^{- di} }{\displaystyle V \beta_v + n_{k}^{- di}} \cdot % \frac{\displaystyle n^{- di}_{k} + \gamma_1 }{\displaystyle \tilde{n}_{k}^{- di} + \gamma_1 + n^{- di}_{k} + \gamma_2 } \cdot % \left(n_{d{k}}^{- di} + \alpha_{dk} \right) & \ {\rm if \ } s_{di} = 0, \\ \frac{\displaystyle \tilde{\beta}_v + \tilde{n}_{k v}^{- di} }{\displaystyle L_{k} \tilde{\beta}_v + \tilde{n}_{k }^{- di} } \cdot% \frac{\displaystyle \tilde{n}^{ - di}_{k} + \gamma_2 }{\displaystyle \tilde{n}^{- di}_{k} + \gamma_1 + n^{- di}_{k} + \gamma_2 } \cdot % \left(n_{d{k}}^{- di} + \alpha_{dk} \right) & \ {\rm if \ } s_{di} = 1. \end{cases}\label{eq:sample-z-base} \end{align}$$ % We replace some of the integrations with the results of the Collapsed Gibbs Sampling, $$\begin{align} q(z_{di} = k) &\propto \exp \bigg( q(s_{di} = 0) \bigg( \mathbb{E}_q\bigg[ \log \frac{\displaystyle \beta_v + n_{k v}^{- di} }{\displaystyle V \beta_v + n_{k}^{- di}} \bigg] + \mathbb{E}_q\bigg[\log \frac{\displaystyle n^{- di}_{k} + \gamma_1 }{\displaystyle \tilde{n}_{k}^{- di} + \gamma_1 + n^{- di}_{k} + \gamma_2 } \bigg] \bigg) \bigg) \\ &\quad \times \exp \bigg( q(s_{di} = 1) \bigg( \mathbb{E}_q\bigg[ \log \frac{\displaystyle \tilde{\beta}_v + \tilde{n}_{k v}^{- di} }{\displaystyle L_{k} \tilde{\beta}_v + \tilde{n}_{k v}^{- di} } \bigg] + \mathbb{E}_q\bigg[ \log \frac{\displaystyle \tilde{n}^{ - di}_{k} + \gamma_2 }{\displaystyle \tilde{n}^{- di}_{k} + \gamma_1 + n^{- di}_{k} + \gamma_2 } \bigg] \bigg) \bigg) \\ &\quad \times \exp \bigg( \mathbb{E}_q\big[ \log ( n_{d{k}}^{- di} + \alpha_{k} ) \big] \bigg) \\ &= \frac{\exp \big[ q(s_{di} = 0) \mathbb{E}_q[\log (\beta_v + n_{k v}^{- di}) ] \big] }{\exp \big[ q(s_{di} = 0) \mathbb{E}_q[\log ( V \beta_v + n_{k}^{- di} )] \big]} \times \frac{\exp \big[ q(s_{di} = 0) \mathbb{E}_q[\log (n^{- di}_{k} + \gamma_1)] \big]}{ \exp \big[ q(s_{di} = 0) \mathbb{E}_q[\log (\tilde{n}_{k}^{- di} + \gamma_1 + n^{- di}_{k} + \gamma_2)] \big]} \\ &\quad \times \frac{\exp \big[ q(s_{di} = 1) \mathbb{E}_q[\log (\tilde{\beta}_v + \tilde{n}_{k v}^{- di})] \big]}{ \exp \big[ q(s_{di} = 1) \mathbb{E}_q[\log (L_{k} \tilde{\beta}_v + \tilde{n}_{k v}^{- di})] \big]} \times \frac{ \exp \big[ q(s_{di} = 1) \mathbb{E}_q[\log (\tilde{n}^{ - di}_{k} + \gamma_2)] \big]} {\exp \big[ q(s_{di} = 1) \mathbb{E}_q[\log (\tilde{n}^{- di}_{k} + \gamma_1 + n^{- di}_{k} + \gamma_2)] \big]} \\ &\quad \times \exp \bigg( \mathbb{E}_q\big[ \log ( n_{d{k}}^{- di} + \alpha_{k} ) \big] \bigg) \end{align}$$

Note that $$\begin{align} \exp \bigg(c \log \frac{a}{b} \bigg) = \exp (c\log a - c\log b) = \frac{\exp(c \log a)}{\exp(c \log b)}. \end{align}$$

Next, we approximate expectations. We use Taylor expansion around $a$, $$\begin{align} \log x \approx \log a + \frac{1}{a} (x-a). \end{align}$$ If $a = \mathbb{E}[x]$, $$\begin{align} \mathbb{E}[\log x] &\approx \mathbb{E}\bigg[ \log \mathbb{E}[x] + \frac{1}{\mathbb{E}[x]} (x - \mathbb{E}[x]) \bigg] = \log \mathbb{E}[x] + \frac{1}{\mathbb{E}[x]} (\mathbb{E}[x] - \mathbb{E}[x]) = \log \mathbb{E}[x], \end{align}$$ which is called CVB0. For example $$\begin{align} \mathbb{E}_{q}[\log ( \beta_v + n_{k v}^{- di}) ] &= \log \mathbb{E}_{q}[\beta_v + n_{k v}^{- di}] = \log ( \beta_v + \mathbb{E}_{q}[n_{k v}^{- di}] ). \end{align}$$

Hence, $$\begin{align} & q(z_{di} = k) \propto \frac{\exp \big[ q(s_{di} = 0) \log (\mathbb{E}_{q}[n_{k v}^{- di} ] + \beta_v) \big] }{\exp \big[ q(s_{di} = 0) \log(\mathbb{E}_{q} [ n_{k}^{- di} ] + V \beta_v) \big]} \times \frac{\exp\big[ q(s_{di} = 0) \log(\mathbb{E}_{q} [n^{- di}_{k} ] + \gamma_1) \big]}{\exp \big[ q(s_{di} = 0) \log(\mathbb{E}_{q} [ \tilde{n}_{k}^{- di} + n^{- di}_{k}] + \gamma_1 + \gamma_2) \big]} \\ &\quad \times \frac{\exp \big[ q(s_{di} = 1) \log (\mathbb{E}_{q}[ \tilde{n}_{k v}^{- di} ] + \tilde{\beta}_v) \big]}{ \exp \big[ q(s_{di} = 1) \log (\mathbb{E}_{q}[ \tilde{n}_{k}^{- di} ] + L_{k} \tilde{\beta}_v) \big]} \times \frac{ \exp \big[ q(s_{di} = 1)\log(\mathbb{E}_{q}[ \tilde{n}^{ - di}_{k} ] + \gamma_2)] \big]} {\exp \big[ q(s_{di} = 1) \log (\mathbb{E}_{q}[ \tilde{n}^{- di}_{k} + n^{- di}_{k} ] + \gamma_1 + \gamma_2) \big]} \\ &\quad \times \big( \mathbb{E}_{q} [n_{d{k}}^{- di} ] + \alpha_{dk} \big) \\ &= \exp \bigg[ q(s_{di} = 0) \bigg( \log (\mathbb{E}_{q}[n_{k v}^{- di} ] + \beta_v) - \log(\mathbb{E}_{q} [ n_{k}^{- di} ] + V \beta_v) + \log(\mathbb{E}_{q} [n^{- di}_{k} ] + \gamma_1) - \log(\mathbb{E}_{q} [ \tilde{n}_{k}^{- di} + n^{- di}_{k}] + \gamma_1 + \gamma_2) \bigg) \\ &\quad + q(s_{di} = 1) \bigg( \log (\mathbb{E}_{q}[ \tilde{n}_{k v}^{- di} ] + \tilde{\beta}_v) - \log (\mathbb{E}_{q}[ \tilde{n}_{k}^{- di} ] + L_{k} \tilde{\beta}_v) + \log(\mathbb{E}_{q}[ \tilde{n}^{ - di}_{k} ] + \gamma_2) - \log (\mathbb{E}_{q}[ \tilde{n}^{- di}_{k} + n^{- di}_{k} ] + \gamma_1 + \gamma_2) \bigg) \bigg] \\ &\quad \times \big( \mathbb{E}_{q} [n_{d{k}}^{- di} ] + \alpha_{dk} \big) \end{align}$$ where $$\begin{align} %%%%%%%%%%%% \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})}[n_{k v}^{- di}] &= \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})} \bigg[\sum_{d=1}^{D}\sum_{i'\neq i}^{N_d} \unicode{x1D7D9}(z_{di'} = k) \unicode{x1D7D9}(s_{di'} = 0) \unicode{x1D7D9}(w_{di'} = v) \bigg] \\ &= \sum_{d=1}^{D}\sum_{i'\neq i}^{N_d} \mathbb{E}_{q(\mathbf{z}^{-di})}[\unicode{x1D7D9}(z_{di'} = k)] \mathbb{E}_{q(\mathbf{s})}[\unicode{x1D7D9}(s_{di'} = 0)] \unicode{x1D7D9}(w_{di'} = v)\\ &= \sum_{d=1}^{D}\sum_{i'\neq i}^{N_d} q(z_{di'} = k) q(s_{di'} = 0) \unicode{x1D7D9}(w_{di'} = v) \\ %%%%%%%%%%%% \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})}[\tilde{n}_{k v}^{- di}] &= \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})} \bigg[\sum_{d=1}^{D}\sum_{i\neq i'}^{N_d} \unicode{x1D7D9}(z_{di'} = k) \unicode{x1D7D9}(s_{di'} = 1) \unicode{x1D7D9}(w_{di'} = v) \bigg] \\ &= \sum_{d=1}^{D}\sum_{i'\neq i}^{N_d} \mathbb{E}_{q(\mathbf{z}^{-di})}[\unicode{x1D7D9}(z_{di'} = k)] \mathbb{E}_{q(\mathbf{s})}[\unicode{x1D7D9}(s_{di'} = 1)] \unicode{x1D7D9}(w_{di'} = v)\\ &= \sum_{d=1}^{D}\sum_{i'\neq i}^{N_d} q(z_{di'} = k) q(s_{di'} = 1) \unicode{x1D7D9}(w_{di'} = v)\\ %%%%%%%%%%%% \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})}[{n}_{k}^{- di}] &= \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})} \bigg[\sum_{d=1}^{D}\sum_{i' \neq i}^{N_d} \unicode{x1D7D9}(z_{di'} = k) \unicode{x1D7D9}(s_{di'} = 0) \bigg] \\ &= \sum_{d=1}^{D}\sum_{i'\neq i}^{N_d} q(z_{di'} = k) q(s_{di'} = 0) \\ %%%%%%%%%%%% \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})}[\tilde{n}_{k}^{- di}] &= \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})} \bigg[\sum_{d=1}^{D}\sum_{i' \neq i}^{N_d} \unicode{x1D7D9}(z_{di'} = k) \unicode{x1D7D9}(s_{di'} = 1) \bigg] \\ &= \sum_{d=1}^{D}\sum_{i'\neq i}^{N_d} q(z_{di'} = k) q(s_{di'} = 1) \\ %%%%%%%%%%%% \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})} [n_{d{k}}^{- di}] &= \mathbb{E}_{q(\mathbf{z}^{-di}) q(\mathbf{s})} \bigg[ \sum_{i' \neq i}^{N_d} \unicode{x1D7D9}(z_{di'} = k) \bigg] \\ &= \sum_{i' \neq i}^{N_d} q(z_{di'} = k) %%%%%%%%%%%% % \E_{q(\bzremove) q(\bs)} [\tilde{n}_{d{k}}^{- di}] &= \E_{q(\bzremove) q(\bs)} \bigg[ \sum_{i' \neq i}^{N_d} \I(z_{di'} = k) \I(s_{di'} = 1) \bigg] \\ % &= \sum_{i' \neq i}^{N_d} q(z_{di'} = k) q(s_{di'} = 1) \end{align}$$

Update $q(\mathbf{s})$

Extract terms related to $q(s_{di})$ from ELBO. $$\begin{align} \mathcal{L}[q(s_{di})] &= \sum_{\mathbf{z}} q(z_{di}) q(\mathbf{z}^{-di}) q(s_{di}) q(\mathbf{s}^{-di}) \log \frac{p(w_{di}, z_{di}, s_{di}\mid \mathbf{z}^{-di}, \mathbf{s}^{-di}, \boldsymbol{\gamma})}{q(s_{di})} \end{align}$$

Results of the Collapsed Gibbs Sampling show, $$\begin{align} \Pr(s_{di} = s \mid \mathbf{s}^{- di}, \mathbf{z}, \mathbf{w}, \boldsymbol{\beta}, \tilde{\boldsymbol{\beta}}, \boldsymbol{\gamma}) & \ \propto \ \begin{cases} \frac{\displaystyle \beta_v + n_{z_{di}, v}^{- di} }{ \displaystyle V \beta_v + n_{z_{di}}^{ - di} } \cdot % (\displaystyle n^{- di}_{z_{di}} + \gamma_1 ) % & {\rm if} \quad s = 0, \\ \frac{\displaystyle \tilde{\beta}_v + \tilde{n}_{z_{di}, v}^{- di} }{\displaystyle L_{z_{di}} \tilde{\beta}_v + \tilde{n}_{z_{di}}^{- di} } \cdot% (\displaystyle \tilde{n}^{- di}_{z_{di}} + \gamma_2 ) % & {\rm if} \quad s = 1. \end{cases} \end{align}$$ \end{align}

From the Variational Bayes, $$\begin{align} q(s_{di} = 0) &\propto \exp \bigg( \sum_{k=1}^{K}q(z_{di} = k) \bigg[ \int q({\phi}_{k, w_{di}}) \log {\phi}_{k, w_{di}}\ d{\phi}_{k, w_{di}} + \int q(\pi_k) \log \pi_k \ d\pi_k \bigg] \bigg) \\ q(s_{di} = 1) &\propto \exp \bigg( \sum_{k=1}^{K}q(z_{di} = k) \bigg[ \int q(\tilde{\phi}_{k, w_{di}}) \log \tilde{\phi}_{k, w_{di}}\ d\tilde{\phi}_{k, w_{di}} + \int q(\pi_k) \log \pi_k \ d\pi_k \bigg] \bigg) \end{align}$$

We replace the some parameters with the results of the Collapsed Gibbs Sampling and the approximate the expectations, $$\begin{align*} q(s_{di} = 0) &\propto \exp \bigg[ \sum_{k=1}^{K}q(z_{di} = k) \bigg( \log (\mathbb{E}_{q(\mathbf{z}) q(\mathbf{s}^{-di})}[ n_{k v}^{- di} ]+ \beta_v) - \log (\mathbb{E}_{q(\mathbf{z}) q(\mathbf{s}^{-di})}[ n_{k}^{ - di} ] + V \beta_v) + \log ( \mathbb{E}_{q(\mathbf{z}) q(\mathbf{s}^{-di})}[ n^{- di}_{k}] + \gamma_1 ) \bigg) \bigg] \\ q(s_{di} = 1) &\propto \exp \bigg[ \sum_{k=1}^{K}q(z_{di} = k) \bigg( \log (\mathbb{E}_{q(\mathbf{z}) q(\mathbf{s}^{-di})}[ \tilde{n}_{k v}^{- di} ]+ \tilde{\beta}_v) - \log (\mathbb{E}_{q(\mathbf{z}) q(\mathbf{s}^{-di})}[ \tilde{n}_{k}^{ - di} ] + L_k \tilde{\beta}_v) + \log ( \mathbb{E}_{q(\mathbf{z}) q(\mathbf{s}^{-di})}[ \tilde{n}^{- di}_{k}] + \gamma_2) \bigg) \bigg] \end{align*}$$

Calculating perplexity

We cannot calculate the log-likelihood explicityly, so we check the approximated perplexity instead. $$\begin{align*} &p(w_{di}^{*} = v\mid \mathbf{w}, \boldsymbol{\alpha}, \boldsymbol{\beta}, \tilde{\boldsymbol{\beta}}, \boldsymbol{\gamma}) \\ &= \sum_{\mathbf{z}, \mathbf{s}} \int p(w_{di}^{*} = v, \mathbf{z}, \mathbf{s}, \boldsymbol{\theta}, \boldsymbol{\phi}, \tilde{\boldsymbol{\phi}}, \boldsymbol{\pi}\mid \mathbf{w}, \boldsymbol{\alpha}, \boldsymbol{\beta}, \tilde{\boldsymbol{\beta}}, \boldsymbol{\gamma}) d\boldsymbol{\phi}d\tilde{\boldsymbol{\phi}} d\boldsymbol{\theta}d\boldsymbol{\pi}\\ &\approx \sum_{\mathbf{z}, \mathbf{s}} \int p(w_{di}^{*} = v, \mathbf{z}, \mathbf{s}\mid \boldsymbol{\theta}, \boldsymbol{\phi}, \tilde{\boldsymbol{\phi}}, \boldsymbol{\pi}) q(\boldsymbol{\theta}, \boldsymbol{\phi}, \tilde{\boldsymbol{\phi}}, \boldsymbol{\pi}) d\boldsymbol{\phi}d\tilde{\boldsymbol{\phi}} d\boldsymbol{\theta}d\boldsymbol{\pi}\\ &\approx \sum_{k=1}^{K}\int p(w_{di}^{*} = v, z_{di} = k\mid \boldsymbol{\theta}, \boldsymbol{\phi}, s_{di} = 0) q(s_{di} = 0\mid \boldsymbol{\pi}) q(\boldsymbol{\theta}) q(\boldsymbol{\phi}) d\boldsymbol{\phi}d\boldsymbol{\theta}d\boldsymbol{\pi}\\ &\quad\quad + \sum_{k=1}^{K}\int p(w_{di}^{*} = v, z_{di} = k\mid \boldsymbol{\theta}, \tilde{\boldsymbol{\phi}}, s_{di} = 1) q(s_{di} = 1\mid \boldsymbol{\pi}) q(\boldsymbol{\theta}) q(\tilde{\boldsymbol{\phi}}) d\tilde{\boldsymbol{\phi}} d\boldsymbol{\theta}d\boldsymbol{\pi}\\ &= \sum_{k=1}^{K}\int q(\boldsymbol{\phi}_{k}) \phi_{kv} d\phi_{k} \int q(\theta_{d}) \theta_{dk} d\theta_{d} \int q(s_{di} = 0\mid \pi_k) \pi_k d\pi_k + \sum_{k=1}^{K}\int q(\tilde{\boldsymbol{\phi}}_{k}) \tilde{\boldsymbol{\phi}}_{kv} d\tilde{\boldsymbol{\phi}}_{k} \int q(\theta_{d}) \theta_{dk} d\theta_{d} \int q(s_{di} = 1\mid \pi_k) \pi_k d\pi_k \\ &= \sum_{k=1}^{K}\bigg[ \frac{\displaystyle \mathbb{E}_{q}[n_{k v} ] + \beta_v }{\displaystyle \mathbb{E}_{q} [ n_{k} ] + V \beta_v} \cdot \dfrac{\mathbb{E}_{q} [n_{d{k}} ] + \alpha_{dk}}{ n_d + \sum_{k'=1}^K \alpha_{dk}} \cdot \frac{\displaystyle \mathbb{E}_q[n_k] + \gamma_2 }{\displaystyle \mathbb{E}_q[\tilde{n}_k] + \gamma_1 + \mathbb{E}_q[n_k] + \gamma_2 } + \frac{\displaystyle \mathbb{E}_{q}[\tilde{n}_{k v} ] + \beta_v }{\displaystyle \mathbb{E}_{q} [ \tilde{n}_{k} ] + L_k \beta_v} \cdot \dfrac{\mathbb{E}_{q} [{n}_{d{k}} ] + \alpha_{dk}}{ {n}_d + \sum_{k'=1}^K \alpha_{dk}} \cdot \frac{\displaystyle \mathbb{E}_q[\tilde{n}_k] + \gamma_1 }{\displaystyle \mathbb{E}_q[\tilde{n}_k] + \gamma_1 + \mathbb{E}_q[n_k] + \gamma_2 } \bigg] \end{align*}$$