diff --git a/paper/rsabsence.Rnw b/paper/rsabsence.Rnw
index 556f1496953a3d9b72c5dfb811752e1f8879be00..9944c1379690f5e7effbcc876ba470ef4dc6930b 100755
--- a/paper/rsabsence.Rnw
+++ b/paper/rsabsence.Rnw
@@ -8,10 +8,14 @@
\usepackage{doi}
\usetikzlibrary{decorations.pathreplacing,calligraphy} % for tikz curly braces
\usepackage{todonotes}
+\usepackage{nameref}
+\usepackage{caption}
\definecolor{darkblue2}{HTML}{273B81}
\definecolor{darkred2}{HTML}{D92102}
+\fboxsep=20pt % for Box
+
\title{Replication of ``null results'' -- Absence of evidence or evidence of
absence?}
@@ -861,68 +865,71 @@ or absence of an effect. It seems logically questionable to declare an
inconclusive replication of an inconclusive original study as a replication
success. While it is important to replicate original studies with null results,
our analysis highlights that they should be analyzed and interpreted
-appropriately. Table~\ref{tab:recommendations} summarizes our recommendations.
+appropriately. Box~\hyperref[box:recommendations]{1} summarizes our
+recommendations.
-\begin{table}[!htb]
+\begin{table}[!h]
\centering
- \caption{Recommendations for the analysis of replication studies of original
- null results. Calculations are based on effect estimates $\hat{\theta}_{i}$
- with standard errors $\sigma_{i}$ for $i \in \{o, r\}$ from an original
- study (subscript $o$) and its replication (subscript $r$). Both effect
- estimates are assumed to be normally distributed around the true effect size
- $\theta$ with known variance $\sigma^{2}$. The effect size $\theta_{n}$
- represents the value of no effect, typically $\theta_{n} = 0$.}
- \label{tab:recommendations}
- \begin{tabular}{p{0.95\textwidth}}
- \toprule
- \textbf{Equivalence test}
- \begin{enumerate}
- \item Specify a margin $\Delta > 0$ that defines an equivalence range
- $[\theta_{n} - \Delta, \theta_{n} + \Delta]$ in which effects are
- considered absent for practical purposes.
- \item Compute the TOST $p$-values for original and replication data
- $$p_{\text{TOST}i}
- = \max\left\{\Phi\left(\frac{\hat{\theta}_{i} - \theta_{n} - \Delta}{\sigma_{i}}\right),
+ \caption*{Box 1: Recommendations for the analysis of replication studies of
+ original null results. Calculations are based on effect estimates
+ $\hat{\theta}_{i}$ with standard errors $\sigma_{i}$ for $i \in \{o, r\}$
+ from an original study (subscript $o$) and its replication (subscript $r$).
+ Both effect estimates are assumed to be normally distributed around the true
+ effect size $\theta$ with known variance $\sigma^{2}$. The effect size
+ $\theta_{n}$ represents the value of no effect, typically $\theta_{n} = 0$.}
+ \label{box:recommendations}
+ \fbox{
+ \begin{tabular}{p{0.875\textwidth}}
+ % \toprule
+ \textbf{Equivalence test}
+ \begin{enumerate}
+ \item Specify a margin $\Delta > 0$ that defines an equivalence range
+ $[\theta_{n} - \Delta, \theta_{n} + \Delta]$ in which effects are
+ considered absent for practical purposes.
+ \item Compute the TOST $p$-values for original and replication data
+ $$p_{\text{TOST}i}
+ = \max\left\{\Phi\left(\frac{\hat{\theta}_{i} - \theta_{n} - \Delta}{\sigma_{i}}\right),
1 - \Phi\left(\frac{\hat{\theta}_{i} - \theta_{n} + \Delta}{\sigma_{i}}\right)\right\},
- ~ i \in \{o, r\}$$
- with $\Phi(\cdot)$ the cumulative distribution function of the
- standard normal distribution.
- \item Declare replication success at level $\alpha$ if
- $p_{\text{TOST}o} \leq \alpha$ and $p_{\text{TOST}r} \leq \alpha$,
- conventionally $\alpha = 0.05$.
- \item Perform a sensitivity analysis with respect to the margin $\Delta$.
- For example, visualize the TOST $p$-values for different margins to
- assess the robustness of the conclusions.
- \end{enumerate} \\
- \midrule \textbf{Bayes factor}
- \begin{enumerate}
- \item Specify a prior distribution for the effect size $\theta$ that
- represents plausible values under the alternative hypothesis that
- there is an effect ($H_{1}\colon \theta \neq \theta_{n})$. For
- example, specify the mean $m$ and variance $v$ of a normal
- distribution $\theta \given H_{1} \sim \Nor(m ,v)$.
- \item Compute the Bayes factors contrasting
- $H_{0} \colon \theta = \theta_{n}$ to
- $H_{1} \colon \theta \neq \theta_{n}$ for original and replication
- data. Assuming a normal prior distribution,
- % $\theta \given H_{1} \sim \Nor(m ,v)$,
- the Bayes factor is
- $$\BF_{01i}
- = \sqrt{1 + \frac{v}{\sigma^{2}_{i}}} \, \exp\left[-\frac{1}{2} \left\{\frac{(\hat{\theta}_{i} -
- \theta_{n})^{2}}{\sigma^{2}_{i}} - \frac{(\hat{\theta}_{i} - m)^{2}}{\sigma^{2}_{i} + v}
+ ~ i \in \{o, r\}$$
+ with $\Phi(\cdot)$ the cumulative distribution function of the
+ standard normal distribution.
+ \item Declare replication success at level $\alpha$ if
+ $p_{\text{TOST}o} \leq \alpha$ and $p_{\text{TOST}r} \leq \alpha$,
+ conventionally $\alpha = 0.05$.
+ \item Perform a sensitivity analysis with respect to the margin $\Delta$.
+ For example, visualize the TOST $p$-values for different margins to
+ assess the robustness of the conclusions.
+ \end{enumerate} \\
+ % \midrule
+ \textbf{Bayes factor}
+ \begin{enumerate}
+ \item Specify a prior distribution for the effect size $\theta$ that
+ represents plausible values under the alternative hypothesis that
+ there is an effect ($H_{1}\colon \theta \neq \theta_{n})$. For
+ example, specify the mean $m$ and variance $v$ of a normal
+ distribution $\theta \given H_{1} \sim \Nor(m ,v)$.
+ \item Compute the Bayes factors contrasting
+ $H_{0} \colon \theta = \theta_{n}$ to
+ $H_{1} \colon \theta \neq \theta_{n}$ for original and replication
+ data. Assuming a normal prior distribution,
+ % $\theta \given H_{1} \sim \Nor(m ,v)$,
+ the Bayes factor is
+ $$\BF_{01i}
+ = \sqrt{1 + \frac{v}{\sigma^{2}_{i}}} \, \exp\left[-\frac{1}{2} \left\{\frac{(\hat{\theta}_{i} -
+ \theta_{n})^{2}}{\sigma^{2}_{i}} - \frac{(\hat{\theta}_{i} - m)^{2}}{\sigma^{2}_{i} + v}
\right\}\right], ~ i \in \{o, r\}.$$
- \item Declare replication success at level $\gamma > 1$ if
- $\BF_{01o} \geq \gamma$ and $\BF_{01r} \geq \gamma$, conventionally
- $\gamma = 3$ (substantial evidence) or $\gamma = 10$ (strong
- evidence).
- \item Perform a sensitivity analysis with respect to the prior
- distribution. For example, visualize the Bayes factors for different
- prior standard deviations to assess the robustness of the
- conclusions.
- \end{enumerate}
- \\
- \bottomrule
- \end{tabular}
+ \item Declare replication success at level $\gamma > 1$ if
+ $\BF_{01o} \geq \gamma$ and $\BF_{01r} \geq \gamma$, conventionally
+ $\gamma = 3$ (substantial evidence) or $\gamma = 10$ (strong
+ evidence).
+ \item Perform a sensitivity analysis with respect to the prior
+ distribution. For example, visualize the Bayes factors for different
+ prior standard deviations to assess the robustness of the
+ conclusions.
+ \end{enumerate}
+ % \\ \bottomrule
+ \end{tabular}
+ }
\end{table}
For both the equivalence testing and the Bayes factor approach, it is critical
diff --git a/rsabsence.pdf b/rsabsence.pdf
old mode 100644
new mode 100755
index 7a00c3ca91f3a19ed11cce801c92c925de2c2fc9..0d149fea0aef339acf6cfdd5c92f5dd5ed99dfde
Binary files a/rsabsence.pdf and b/rsabsence.pdf differ