diff --git a/paper/rsabsence.Rnw b/paper/rsabsence.Rnw index 20bd4e4240aadf583ecb3ce4e03add294b076910..00324ab30d2403767dcd56d6e76bd873064f5756 100755 --- a/paper/rsabsence.Rnw +++ b/paper/rsabsence.Rnw @@ -844,7 +844,7 @@ pTOSTa <- function(estimate, se, null = 0, margin) { @ << "pTOST-more-educational-version", eval = FALSE, echo = TRUE, size = "small" >>= ## R function to compute TOST p-value based on effect estimate, standard error, -## null value (default is 0), equivalence margin +## null value (default is 0), and equivalence margin pTOST <- function(estimate, se, null = 0, margin) { p1 <- pnorm(q = (estimate - null - margin) / se) p2 <- 1 - pnorm(q = (estimate - null + margin) / se) @@ -866,8 +866,8 @@ pTOST <- function(estimate, se, null = 0, margin) { \item Specify a prior distribution for the effect size $\theta$ that represents plausible values under the alternative hypothesis that there is an effect ($H_{1}\colon \theta \neq \theta_{0})$. For - example, specify the mean $m$ and variance $v$ of a normal - distribution $\theta \given H_{1} \sim \Nor(m ,v)$. + example, specify the mean $m$ and standard deviation $s$ of a normal + distribution $\theta \given H_{1} \sim \Nor(m, s^{2})$. \item Compute the Bayes factors contrasting $H_{0} \colon \theta = \theta_{0}$ to $H_{1} \colon \theta \neq \theta_{0}$ for original and replication @@ -875,25 +875,27 @@ pTOST <- function(estimate, se, null = 0, margin) { % $\theta \given H_{1} \sim \Nor(m ,v)$, the Bayes factor is $$\BF_{01,i} - = \sqrt{1 + \frac{v}{\sigma^{2}_{i}}} \, \exp\left[-\frac{1}{2} \left\{\frac{(\hat{\theta}_{i} - - \theta_{0})^{2}}{\sigma^{2}_{i}} - \frac{(\hat{\theta}_{i} - m)^{2}}{\sigma^{2}_{i} + v} + = \sqrt{1 + \frac{s^{2}}{\sigma^{2}_{i}}} \, \exp\left[-\frac{1}{2} \left\{\frac{(\hat{\theta}_{i} - + \theta_{0})^{2}}{\sigma^{2}_{i}} - \frac{(\hat{\theta}_{i} - m)^{2}}{\sigma^{2}_{i} + s^2} \right\}\right], ~ i \in \{o, r\}.$$ \begin{minipage}[c]{0.95\linewidth} << "BF01-version-that-we-used", eval = FALSE, echo = FALSE, size = "small" >>= ## R function to compute Bayes factor based on effect estimate, standard error, -## null value (default is 0), prior mean (default is null value), prior variance -BF01a <- function(estimate, se, null = 0, priormean = null, priorvar) { +## null value (default is 0), prior mean (default is null value), and prior +## standard deviation +BF01a <- function(estimate, se, null = 0, priormean = null, priorsd) { f0 <- dnorm(x = estimate, mean = null, sd = se) - f1 <- dnorm(x = estimate, mean = priormean, sd = sqrt(se^2 + priorvar)) + f1 <- dnorm(x = estimate, mean = priormean, sd = sqrt(se^2 + priorsd^2)) return(f0/f1) } @ << "BF01-more-educational-version", eval = FALSE, echo = TRUE, size = "small" >>= ## R function to compute Bayes factor based on effect estimate, standard error, -## null value (default is 0), prior mean (default is null value), prior variance -BF01 <- function(estimate, se, null = 0, priormean = null, priorvar) { - bf <- sqrt(1 + priorvar/se^2) * exp(-0.5 * ((estimate - null)^2 / se^2 - - (estimate - priormean)^2 / (se^2 + priorvar))) +## null value (default is 0), prior mean (default is null value), and prior +## standard deviation +BF01 <- function(estimate, se, null = 0, priormean = null, priorsd) { + bf <- sqrt(1 + priorsd^2/se^2) * exp(-0.5 * ((estimate - null)^2 / se^2 - + (estimate - priormean)^2 / (se^2 + priorsd^2))) return(bf) } @ @@ -1019,55 +1021,27 @@ data from the RPCB were obtained by downloading the files from relevant variables as indicated in the R script \texttt{preprocess-rpcb-data.R} which is available in our git repository. -\bibliography{bibliography} -\section*{Appendix} -\subsection*{Sensitivity analysis} +\section*{Appendix: Sensitivity analyses} -As discussed before, the post-hoc specification of equivalence margins and the -choice of the prior distribution for the SMD under the alternative $H_{1}$ is -debatable and controversial. Commonly used margins in clinical research are much -more stringent; for instance, in oncology, a margin of $\Delta = \log(1.3)$ is -commonly used for log odds/hazard ratios, whereas in bioequivalence studies a -margin of \mbox{$\Delta = \log(1.25) % = \Sexpr{round(log(1.25), 2)} +As discussed before, the post-hoc specification of equivalence margins $\Delta$ +and prior distribution for the SMD under the alternative $H_{1}$ is debatable. +Commonly used margins in clinical research are much more stringent; for +instance, in oncology, a margin of $\Delta = \log(1.3)$ is commonly used for log +odds/hazard ratios, whereas in bioequivalence studies a margin of +\mbox{$\Delta = \log(1.25) % = \Sexpr{round(log(1.25), 2)} $} is the convention \citep[chapter 22]{Senn2008}. These margins would -translate into margins of $\Delta -= % \log(1.3)\sqrt{3}/\pi = +translate into margins of $\Delta = % \log(1.3)\sqrt{3}/\pi = \Sexpr{round(log(1.3)*sqrt(3)/pi, 2)}$ and $\Delta = % \log(1.25)\sqrt{3}/\pi = \Sexpr{round(log(1.25)*sqrt(3)/pi, 2)}$ on the SMD scale, respectively, using the $\text{SMD} = (\surd{3} / \pi) \log\text{OR}$ conversion \citep[p. -233]{Cooper2019}. As for the Bayesian hypothesis testing we specified a normal +233]{Cooper2019}. Similarly, for the Bayesian factor we specified a normal unit-information prior under the alternative while other normal priors with -smaller/larger standard deviations could have been considered. -In the sensitivity analysis in the top plot of Figure~\ref{fig:sensitivity} we -show the number of successful replications as a function of the margin $\Delta$ -and for different TOST \textit{p}-value thresholds. Such an ``equivalence -curve'' approach was first proposed by \citet{Hauck1986}. We see that for -realistic margins between $0$ and $1$, the proportion of replication successes -remains below $50\%$ for the conventional $\alpha = 0.05$ level. To achieve a -success rate of $11/15 = \Sexpr{round(11/15*100, 0)}\%$, as was achieved with -the non-significance criterion from the RPCB, unrealistic margins of $\Delta > -2$ are required, highlighting the paucity of evidence provided by these studies. -Changing the success criterion to a more lenient level ($\alpha = 0.1$) or a -more stringent level ($\alpha = 0.01$) hardly changes this conclusion. - -In the bottom plot of Figure~\ref{fig:sensitivity} a sensitivity analysis is -reported with respect to the choice of the prior standard deviation and the -Bayes factor threshold. It is uncommon to specify prior standard -deviations larger than the unit-information standard deviation of $2$, as this -corresponds to the assumption of very large effect sizes under the alternatives. -However, to achieve replication success for a larger proportion of replications -than the observed $\Sexpr{bfSuccesses}/\Sexpr{ntotal} = -\Sexpr{round(bfSuccesses/ntotal*100, 0)}\%$, unreasonably large prior standard -deviations have to be specified. For instance, a standard deviation of roughly -$5$ is required to achieve replication success in $50\%$ of the replications at -a lenient Bayes factor threshold of $\gamma = 3$. The standard deviation needs -to be almost $20$ so that the same success rate $11/15 = \Sexpr{round(11/15*100, -0)}\%$ as with the non-significance criterion is achieved. The necessary -standard deviations are even higher for stricter Bayes factor threshold, -such as $\gamma = 6$ or $\gamma = 10$. +smaller/larger standard deviations could have been considered. Here, we +therefore investigate the sensitivity of our conclusions with respect to these +parameters. \begin{figure}[!htb] << "sensitivity", fig.height = 6.5 >>= @@ -1196,6 +1170,39 @@ grid.arrange(plotA, plotB, ncol = 1) \label{fig:sensitivity} \end{figure} +The top plot of Figure~\ref{fig:sensitivity} shows the number of +successful replications as a function of the margin $\Delta$ and for different +TOST \textit{p}-value thresholds. Such an ``equivalence curve'' approach was +first proposed by \citet{Hauck1986}. We see that for realistic margins between +$0$ and $1$, the proportion of replication successes remains below $50\%$ for +the conventional $\alpha = 0.05$ level. To achieve a success rate of +$11/15 = \Sexpr{round(11/15*100, 0)}\%$, as was achieved with the +non-significance criterion from the RPCB, unrealistic margins of $\Delta > 2$ +are required, highlighting the paucity of evidence provided by these studies. +Changing the success criterion to a more lenient level ($\alpha = 0.1$) or a +more stringent level ($\alpha = 0.01$) hardly changes the conclusion. + +The bottom plot of Figure~\ref{fig:sensitivity} shows a sensitivity analysis +regarding the choice of the prior standard deviation and the Bayes factor +threshold. It is uncommon to specify prior standard deviations larger than the +unit-information standard deviation of $2$, as this corresponds to the +assumption of very large effect sizes under the alternatives. However, to +achieve replication success for a larger proportion of replications than the +observed +$\Sexpr{bfSuccesses}/\Sexpr{ntotal} = \Sexpr{round(bfSuccesses/ntotal*100, 0)}\%$, +unreasonably large prior standard deviations have to be specified. For instance, +a standard deviation of roughly $5$ is required to achieve replication success +in $50\%$ of the replications at a lenient Bayes factor threshold of +$\gamma = 3$. The standard deviation needs to be almost $20$ so that the same +success rate $11/15 = \Sexpr{round(11/15*100, 0)}\%$ as with the +non-significance criterion is achieved. The necessary standard deviations are +even higher for stricter Bayes factor threshold, such as $\gamma = 6$ or +$\gamma = 10$. + + +\bibliography{bibliography} + + << "sessionInfo1", eval = Reproducibility, results = "asis" >>= ## print R sessionInfo to see system information and package versions ## used to compile the manuscript (set Reproducibility = FALSE, to not do that) diff --git a/rsabsence.pdf b/rsabsence.pdf index f05d85102bfde82cea1cc6d074b107a138d4b330..e2af408c10eaf97a97eeb24ec5c77005f6cc9426 100644 Binary files a/rsabsence.pdf and b/rsabsence.pdf differ