diff --git a/paper/bibliography.bib b/paper/bibliography.bib
index f1e1cebc32496f26a80556bc39164ecd1a4a64d4..0fc5f1d16d40db92e18c5184804823e083d63603 100755
--- a/paper/bibliography.bib
+++ b/paper/bibliography.bib
@@ -1,3 +1,19 @@
+@book{Wellek2010,
+ title={Testing statistical hypotheses of equivalence and noninferiority},
+ author={Wellek, Stefan},
+ year={2010},
+ publisher={CRC press}
+}
+
+@article{Makin2019,
+ doi = {10.7554/elife.48175},
+ year = {2019},
+ volume = {8},
+ author = {Tamar R Makin and Jean-Jacques Orban de Xivry},
+ title = {Ten common statistical mistakes to watch out for when writing or reviewing a manuscript},
+ journal = {{eLife}}
+}
+
@article{Campbell2021,
doi = {10.15626/mp.2020.2506},
year = {2021},
@@ -147,7 +163,7 @@
journal = {Psychological Methods}
}
-@article{Micheloud2022,
+@misc{Micheloud2022,
doi = {10.48550/ARXIV.2204.06960},
author = {Micheloud, Charlotte and Held, Leonhard},
title = {The replication of non-inferiority and equivalence studies},
@@ -721,7 +737,7 @@ doi = {10.1080/00031305.2019.1583913}
copyright = {Creative Commons Attribution 4.0 International}
}
-@article{Pawel2022c,
+@misc{Pawel2022c,
doi = {10.48550/ARXIV.2211.02552},
author = {Pawel, Samuel and Consonni, Guido and Held, Leonhard},
title = {Bayesian approaches to designing replication studies},
@@ -1096,9 +1112,9 @@ url = {www.fda.gov/regulatory-information/search-fda-guidance-documents/providi
title = {New preprint server for medical research},
journal = {{BMJ}}
}
+
@book{NSF2019,
doi = {10.17226/25303},
- url = {https://doi.org/10.17226/25303},
year = {2019},
month = sep,
publisher = {National Academies Press},
@@ -2108,7 +2124,7 @@ journal = {{eLife}}
@Article{Camerer2018,
authorfull = {C. F. Camerer and A. Dreber and F. Holzmeister and T. Ho and J. Huber and M. Johannesson and M. Kirchler and G. Nave and B. Nosek and T. Pfeiffer and A. Altmejd and N. Buttrick and T. Chan and Y. Chen and E. Forsell and A. Gampa and E. Heikenstein and L. Hummer and T. Imai and S. Isaksson and D. Manfredi and J. Rose and E. Wagenmakers and H. Wu},
author = {C. F. Camerer and A. Dreber and F. Holzmeister and T. Ho and J. Huber and M. Johannesson and M. Kirchler and G. Nave and B. Nosek and others},
- title = {Evaluating the replicability of social science experiments in {Nature} and {Science} between 2010 and 2015},
+ title = {Evaluating the replicability of social science experiments in Nature and Science between 2010 and 2015},
journal = {Nature Human Behavior},
year = {2018},
volume = {2},
diff --git a/paper/elife.cls b/paper/elife.cls
new file mode 100644
index 0000000000000000000000000000000000000000..cfff953a8d742ceb400ba1c30e994e48932397c1
--- /dev/null
+++ b/paper/elife.cls
@@ -0,0 +1,529 @@
+% A template for eLife submissions.
+%
+% Created by Overleaf (writeLaTeX).
+%
+\NeedsTeXFormat{LaTeX2e}
+\ProvidesClass{elife}[2022/06/06, v1.11]
+
+%% Move all option processing before loading class
+\newif\if@reqslineno
+\DeclareOption{lineno}{\@reqslinenotrue}
+
+%% Linespacing.
+\newif\if@onehalfspacing
+\newif\if@doublespacing
+\DeclareOption{onehalfspacing}{\@onehalfspacingtrue}
+\DeclareOption{doublespacing}{\@doublespacingtrue}
+
+\DeclareOption*{\PassOptionsToClass{\CurrentOption}{extarticle}}
+\ExecuteOptions{}
+\ProcessOptions\relax
+\LoadClass{extarticle}
+
+\RequirePackage[english]{babel}
+
+\RequirePackage{calc}
+\RequirePackage{etoolbox}
+\RequirePackage{regexpatch}
+\RequirePackage{ifxetex,ifluatex}
+
+\newif\ifxetexorluatex
+\ifxetex
+ \xetexorluatextrue
+\else
+ \ifluatex
+ \xetexorluatextrue
+ \else
+ \xetexorluatexfalse
+ \fi
+\fi
+
+\RequirePackage{amsmath}
+\RequirePackage{mdframed}
+
+\RequirePackage{lineno}
+\if@reqslineno\linenumbers\fi
+
+% Note v1.0 of elife.cls used Trueno font; this was
+% changed to use Open Sans in v1.1
+\ifxetexorluatex
+ \RequirePackage{unicode-math}
+ % v1.7: xits-math font name changed to XITSMath in TL2019
+ \RequirePackage[default]{opensans}
+ \setmonofont[Scale=MatchUppercase]{Latin Modern Mono}
+ % XITSMath is really STIX. Really.
+ % https://tex.stackexchange.com/questions/227216/stix-versus-xits
+ \setmathfont[Extension = .otf,BoldFont = XITSMath-Bold,]{XITSMath-Regular}
+\else
+ \RequirePackage[T1]{fontenc}
+ \RequirePackage[utf8]{inputenc}
+ \RequirePackage{stix}
+ \RequirePackage[default]{opensans}
+ \renewcommand{\ttdefault}{lmtt}
+\fi
+
+\RequirePackage{microtype}
+
+% Trueno/Open Sans requires a bigger "single" linespread.
+\linespread{1.2}
+\if@onehalfspacing\linespread{1.5}\fi
+\if@doublespacing\linespread{2.0}\fi
+
+
+\usepackage[dvipsnames]{xcolor}
+\RequirePackage{graphicx}
+\definecolor{eLifeDarkBlue}{HTML}{273B81}
+\definecolor{eLifeLightBlue}{HTML}{0A9DD9}
+\definecolor{eLifeMediumGrey}{HTML}{6D6E70}
+\definecolor{eLifeLightGrey}{HTML}{929497}
+
+\RequirePackage{booktabs}
+\RequirePackage{authblk}
+
+\RequirePackage[left=6cm,%
+ marginparwidth=4cm,%
+ marginparsep=0.5cm,%
+ right=1.3cm,%
+ top=2cm,%
+ bottom=2.5cm,%
+ headheight=21pt,%
+ headsep=2\baselineskip,%
+ columnsep=2em,%
+ letterpaper]{geometry}%
+\RequirePackage{changepage}
+
+\RequirePackage{silence}
+\WarningFilter{caption}{The option `hypcap=true' will be ignored}
+
+\RequirePackage[labelfont={bf},%
+ labelsep=period,%
+ justification=raggedright,%
+ singlelinecheck=false,%
+ tableposition=top,font=small]
+ {caption}
+
+% \captionsetup*[table]{skip=\medskipamount}
+
+
+\RequirePackage{natbib}
+\renewcommand{\bibfont}{\small}
+% modifed from https://github.com/gbhutani/vancouver_authoryear_bibstyle/
+\IfFileExists{vancouver-elife.bst}
+ {\bibliographystyle{vancouver-elife}}
+ {\PackageWarning{elife}{vancouver-elife.bst not found; falling back to apalike bibliography style.}\bibliographystyle{apalike}}
+% Make author in citation italic
+\renewcommand{\NAT@nmfmt}[1]{{\bfseries\itshape\color{eLifeMediumGrey} #1}}
+
+% ...as well as the year
+\xpatchcmd{\NAT@citex}
+{\@citea\NAT@hyper@{\NAT@nmfmt{\NAT@nm}\NAT@date}}
+{\@citea\NAT@hyper@{\NAT@nmfmt{\NAT@nm}\NAT@nmfmt{\NAT@date}}}
+{}{\PackageWarning{eLife}{Failed to patch year format in citation}}
+
+\xpatchcmd{\NAT@citex}
+{\else\unskip\NAT@spacechar\NAT@hyper@{\NAT@date}}
+{\else\unskip\NAT@spacechar\NAT@hyper@{\NAT@nmfmt{\NAT@date}}}
+{}{\PackageWarning{eLife}{Failed to patch year format in citation}}
+
+\xpatchcmd{\NAT@citex}
+{\hyper@natlinkbreak{\NAT@aysep\NAT@spacechar}{\@citeb\@extra@b@citeb}\NAT@date}
+{\hyper@natlinkbreak{\NAT@nmfmt{\NAT@aysep\NAT@spacechar}}{\@citeb\@extra@b@citeb}\NAT@nmfmt{\NAT@date}}
+{}{\PackageWarning{eLife}{Failed to patch year format in citation}}
+
+\xpatchcmd{\NAT@citex}
+{\@citea\NAT@hyper@{\NAT@date}}
+{\@citea\NAT@hyper@{\NAT@nmfmt{\NAT@date}}}
+{}{\PackageWarning{eLife}{Failed to patch year format in citation}}
+
+\xpatchcmd{\NAT@citex}
+{{\@citeb\@extra@b@citeb}\NAT@date}
+{{\@citeb\@extra@b@citeb}\NAT@nmfmt{\NAT@date}}
+{}{\PackageWarning{eLife}{Failed to patch year format in citation}}
+%% There, we're finally done with patching the year in citations.
+
+%
+% headers and footers
+%
+\RequirePackage{fancyhdr} % custom headers/footers
+\RequirePackage{lastpage} % Number of pages in the document
+\pagestyle{fancy} % Enables the custom headers/footers
+\addtolength{\headwidth}{\marginparsep}
+\addtolength{\headwidth}{\marginparwidth}
+\fancyhf{}
+% Removing header in v1.9
+% \chead{%
+% \setlength{\fboxsep}{3pt}
+% \colorbox{eLifeMediumGrey}{\begin{minipage}{\headwidth}\centering\color{white} Manuscript submitted to eLife\end{minipage}}%
+% }
+
+% Footers
+\rfoot{\small\color{eLifeMediumGrey}\thepage\space of\space\pageref{LastPage}}%
+\preto{\footrule}{\color{eLifeMediumGrey}}
+\renewcommand{\headrulewidth}{0pt}% % No header rule
+\renewcommand{\footrulewidth}{0.4pt}% % No footer rule
+
+%
+% section/subsection/paragraph set-up
+%
+\setcounter{secnumdepth}{0}
+\RequirePackage[explicit]{titlesec}
+\titleformat{\section}
+ {\color{eLifeMediumGrey}\Large\bfseries}
+ {\thesection}{}{#1}[]
+\titleformat{\subsection}
+ {\large\bfseries}
+ {\thesubsection}{}{#1}[]
+\titleformat{\subsubsection}
+ {\large}
+ {\thesubsubsection}{}{#1}[]
+\titleformat{\paragraph}
+ {\color{eLifeMediumGrey}\large}
+ {\theparagraph}{}{#1}[]
+\titlespacing*{\section}{0pc}{3ex \@plus4pt \@minus3pt}{0pt}
+\titlespacing*{\subsection}{0pc}{2.5ex \@plus3pt \@minus2pt}{0pt}
+\titlespacing*{\subsubsection}{0pc}{2ex \@plus2.5pt \@minus1.5pt}{0pt}
+\titlespacing*{\paragraph}{0pc}{1.5ex \@plus2pt \@minus1pt}{0pt}
+
+\RequirePackage{enumitem}
+\setlist{noitemsep}
+
+\RequirePackage{alphalph}
+\newalphalph{\fnsymbolmult}[mult]{\@fnsymbol}{5}
+
+\newcounter{authorfn}
+\setcounter{authorfn}{1}
+\newcommand{\authfn}[1]{%
+ \fnsymbolmult{\numexpr\value{authorfn}+#1}%
+}
+
+\def\@correspondence{}
+\def\@contribution{}
+\def\@presentaddress{}
+\def\@deceased{}
+
+\newcommand{\corr}[2]{%
+ \ifx\empty\@correspondence\else\appto{\@correspondence}{; }{}{}\fi
+ \appto{\@correspondence}{%
+ \url{#1}%
+ \ifx\empty#2\else\space(#2)\fi
+ }{}{}%
+}
+
+\newcommand{\contrib}[2][]{
+ \appto{\@contribution}{%
+ \ifx\empty#1\else\textsuperscript{#1}\fi
+ #2\\
+ }{}{}
+}
+
+\newcommand{\presentadd}[2][]{
+ \ifx\empty\@presentaddress\else\appto{\@presentaddress}{; }{}{}\fi
+ \appto{\@presentaddress}{%
+ \ifx\empty#1\else\textsuperscript{#1}\fi
+ #2%
+ }{}{}
+}
+
+\newcommand{\deceased}[1]{\def\@deceased{\textsuperscript{#1}Deceased}}
+
+\reversemarginpar
+
+%
+% custom title page
+%
+\renewcommand{\Authfont}{\bfseries\large\raggedright}
+\renewcommand{\Affilfont}{\mdseries\large\raggedright}
+\renewcommand{\Authands}{, }
+\setlength{\affilsep}{16pt}
+\renewcommand{\AB@affilsepx}{; \protect\Affilfont}
+
+\newcommand{\themetadata}{%
+ \textbf{*For correspondence:\\} \@correspondence\par
+ \ifx\empty\@contribution\else
+ \bigskip\@contribution\par\fi
+ \ifx\empty\@presentaddress\else
+ \bigskip\textbf{Present address: }\@presentaddress\par\fi
+ \bigskip
+ \ifx\empty\@deceased\else\@deceased\par\fi
+}
+
+\patchcmd{\@author}{\AB@authlist\\[\affilsep]\AB@affillist}{\AB@authlist\\[\affilsep]
+ \marginpar{\raggedright\footnotesize\themetadata\par}\AB@affillist}{}{}
+
+\renewcommand{\maketitle}{%
+ \vskip36pt%
+ {\color{eLifeDarkBlue}\raggedright\bfseries\fontsize{22}{27}\selectfont \@title\par}%
+ \vskip16pt
+ {\@author\par}
+ \vskip8pt
+}
+
+\renewenvironment{abstract}{%
+ \setlength{\parindent}{0pt}\raggedright
+ \textcolor{eLifeMediumGrey}{\rule{\textwidth}{2pt}}
+ \vskip16pt
+ \textcolor{eLifeLightBlue}{\large\bfseries\abstractname\space}
+}{%
+ \vskip8pt
+ \textcolor{eLifeMediumGrey}{\rule{\textwidth}{2pt}}
+ \vskip16pt
+}
+
+%% Insert a grey line to separate floats from main text
+\newcommand{\topfigrule}{\vskip8pt\noindent{\rule{\linewidth}{1pt}}}
+\newcommand{\botfigrule}{\noindent{\rule{\linewidth}{1pt}}\vskip8pt}
+
+\RequirePackage{newfloat}
+\RequirePackage{wrapfig}
+\AtEndEnvironment{wrapfigure}{\vskip8pt\noindent{\rule{\hsize}{1pt}}}
+% \RequirePackage[lflt]{floatflt}
+% \AtEndEnvironment{floatingfigure}{\vskip8pt\noindent\textcolor{eLifeMediumGrey}{\rule{\hsize}{2pt}}}
+
+\DeclareFloatingEnvironment[placement=btp,name=Box]{featurebox}
+\captionsetup[featurebox]{font={Large,bf,color=eLifeDarkBlue}}
+
+% v1.11 2022-06 Updated many caption casing and dashes
+\newcounter{featurefigure}
+\newcounter{featuretable}
+\AtBeginEnvironment{featurebox}{%
+ \setcounter{featurefigure}{0}%
+ \setcounter{featuretable}{0}%
+ \newcommand{\featurefig}[1]{%
+ \refstepcounter{featurefigure}%
+ \vskip\smallskipamount%
+ {\small\textbf{\color{eLifeDarkBlue}Box \arabic{featurebox}---figure \arabic{featurefigure}.}\space #1\par}\medskip}
+ \newcommand{\featuretable}[1]{%
+ \refstepcounter{featuretable}%
+ \vskip\smallskipamount%
+ {\small\textbf{\color{eLifeDarkBlue}Box \arabic{featurebox}---table \arabic{featuretable}.}\space #1\par}\medskip}
+
+}
+\apptocmd{\featurebox}{%
+ \begin{mdframed}[linewidth=0pt,backgroundcolor=eLifeLightBlue!10,fontcolor=eLifeDarkBlue]
+ \if@reqslineno\addtolength{\linenumbersep}{1em}\internallinenumbers\fi%
+}{}{}
+\pretocmd{\endfeaturebox}{\end{mdframed}}{}{}
+
+
+\newenvironment{fullwidth}{%
+ \begin{adjustwidth}{-4.5cm}{}%
+ \hsize=\linewidth%
+}{\end{adjustwidth}}
+
+\appto{\appendix}{%
+ \captionsetup*[figure]{name={Appendix \arabic{appendix}---figure },font={color=eLifeDarkBlue,small},skip=\smallskipamount}%
+ \captionsetup*[table]{name={Appendix \arabic{appendix}---table },font={color=eLifeDarkBlue,small}}%
+}
+
+\newcounter{figsupp}
+\setcounter{figsupp}{0}
+\newcounter{data}
+\setcounter{data}{0}
+%% Added videos 2021/03/07
+\newcounter{srccode}
+\setcounter{srccode}{0}
+%% Added fig supp data and source code 2022/06
+\newcounter{fsdata}
+\setcounter{fsdata}{0}
+\newcounter{fssrccode}
+\setcounter{fssrccode}{0}
+%% Added videos 2018/06/03
+\newcounter{video}
+\setcounter{video}{0}
+\newcounter{videosupp}
+\setcounter{videosupp}{0}
+\def\supplist{}
+
+\RequirePackage{newfile}
+\newoutputstream{suppinfo}
+\openoutputfile{\jobname.suppinfo}{suppinfo}
+\addtostream{suppinfo}{\protect\setcounter{figure}{0}}
+
+% Added video source data and code 2022/06
+\DeclareRobustCommand{\video}[1]{%
+ \refstepcounter{video}%
+ \setcounter{data}{0}%
+ \setcounter{srccode}{0}%
+ \bigskip%
+ \noindent{\small\textbf{Video \arabic{video}.} #1\par}%
+ \bigskip
+}
+\DeclareRobustCommand{\videodata}[1]{%
+ \refstepcounter{data}%
+ \par\noindent{\small\textbf{Video~\thevideo---source data \arabic{data}.} #1}\par
+}
+\DeclareRobustCommand{\videosrccode}[1]{%
+ \refstepcounter{srccode}%
+ \par\noindent{\small\textbf{Video~\thevideo---source code \arabic{srccode}.} #1}\par
+}
+
+% Added fig supp source data and code 2022/06
+\DeclareRobustCommand{\figsuppsrccode}[1]{%
+ \protect\refstepcounter{fssrccode}%
+ \par\small\textbf{\figurename~\thefigure---figure supplement \arabic{figsupp}---source code~\arabic{fssrccode}. } #1\par
+}
+\DeclareRobustCommand{\figsuppdata}[1]{%
+ \protect\refstepcounter{fsdata}%
+ \par\small\textbf{\figurename~\thefigure---figure supplement \arabic{figsupp}---source data~\arabic{fsdata}. } #1\par
+}
+
+
+\AtBeginEnvironment{figure}{%
+ \setcounter{figsupp}{0}
+ \setcounter{data}{0}
+ \setcounter{videosupp}{0}
+ \setcounter{srccode}{0}
+ \addtostream{suppinfo}{
+ \protect\setcounter{figsupp}{0}
+ \protect\refstepcounter{figure}
+ }
+}
+
+\AtBeginEnvironment{wrapfigure}{%
+ \setcounter{figsupp}{0}
+ \setcounter{data}{0}
+ \setcounter{videosupp}{0}
+ \setcounter{srccode}{0}
+ \addtostream{suppinfo}{
+ \protect\setcounter{figsupp}{0}
+ \protect\refstepcounter{figure}
+ }
+}
+
+%% Updated 2017/06/30 to allow optional argument
+\DeclareRobustCommand{\figsupp}[3][]{%
+ \refstepcounter{figsupp}%
+ \addtostream{suppinfo}{%
+ \protect\refstepcounter{figsupp}
+ \protect\setcounter{fsdata}{0}
+ \protect\setcounter{fssrccode}{0}
+ }%
+ {%
+ \ifstrequal{#1}{none}{}{%
+ \small\textbf{\figurename~\thefigure---figure supplement \arabic{figsupp}.}
+ \ifstrempty{#1}{%
+ \renewcommand\figsuppdata[1]{}%
+ \renewcommand\figsuppsrccode[1]{}%
+ #2}{%
+ \renewcommand\figsuppdata[1]{}%
+ \renewcommand\figsuppsrccode[1]{}%
+ #1}}}\raggedright\par
+ \addtostream{suppinfo}{%
+ \noindent\protect\begin{minipage}{\linewidth}
+ \protect #3\noexpand\par
+ \textbf{\figurename~\thefigure---figure supplement \arabic{figsupp}.} #2\noexpand\par
+ \vskip8pt
+ \protect\end{minipage}
+ \vskip16pt
+ }
+}
+\DeclareRobustCommand{\figdata}[1]{%
+ \refstepcounter{data}
+ {\small\textbf{\figurename~\thefigure---source data \arabic{data}.} #1}\par
+}
+%% Added 2021/03/07
+\DeclareRobustCommand{\figsrccode}[1]{%
+ \refstepcounter{srccode}
+ {\small\textbf{\figurename~\thefigure---source code \arabic{srccode}.} #1}\par
+}
+%% Added 2018/06/03
+\DeclareRobustCommand{\videosupp}[1]{%
+ \refstepcounter{videosupp}
+ {\small\textbf{\figurename~\thefigure---video \arabic{videosupp}.} #1}\par
+}
+
+% Added table source data and code 2022/06
+\AtBeginEnvironment{table}{%
+ \setcounter{data}{0}
+ \setcounter{srccode}{0}
+ \setcounter{videosupp}{0}
+}
+
+\AtBeginEnvironment{wraptable}{%
+ \setcounter{data}{0}
+ \setcounter{srccode}{0}
+ \setcounter{videosupp}{0}
+}
+
+\DeclareRobustCommand{\tabledata}[1]{%
+ \refstepcounter{data}
+ {\small\textbf{\tablename~\thetable---source data \arabic{data}.} #1}\par
+}
+\DeclareRobustCommand{\tablesrccode}[1]{%
+ \refstepcounter{srccode}
+ {\small\textbf{\tablename~\thetable---source code \arabic{srccode}.} #1}\par
+}
+
+
+\AtEndDocument{%
+ \closeoutputstream{suppinfo}
+% \pagestyle{empty}
+ \renewcommand{\footrule}{}
+ \rfoot{}
+ \input{\jobname.suppinfo}
+}
+
+\newcounter{appendix}
+\setcounter{appendix}{0}
+\newenvironment{appendixbox}{%
+ \setcounter{figure}{0}
+ \setcounter{table}{0}
+ \refstepcounter{appendix}%
+ \clearpage%
+ \patchcmd{\ttlf@section}{eLifeMediumGrey}{eLifeDarkBlue}{}{}
+ \noindent{\bfseries\Large\color{eLifeMediumGrey}Appendix \arabic{appendix}\par}
+ \nolinenumbers%
+ \begin{mdframed}[hidealllines=true,backgroundcolor=eLifeLightBlue!10,fontcolor=eLifeDarkBlue,leftline=true,linecolor=eLifeLightBlue,linewidth=1em]
+ \if@reqslineno\addtolength{\linenumbersep}{2em}\internallinenumbers\fi
+}{%
+ \end{mdframed}
+}
+
+\RequirePackage{hyperref}
+\hypersetup{colorlinks=true,allcolors=black,citecolor=eLifeMediumGrey,linkcolor=eLifeMediumGrey,urlcolor=eLifeLightBlue,hypertexnames=false}
+\urlstyle{sf}
+
+% Other desired commands
+\renewcommand{\equationautorefname}{Eq.}
+%% Added autoref support for \figsupp, \figdata, \video, \videosupp, (2021-03-04) \figsrccode
+% Added support for figsupp, video source data and code 2022/06
+\newcommand{\figsuppautorefname}{figure Supplement}
+\newcommand{\dataautorefname}{source data}
+\newcommand{\videoautorefname}{Video}
+\newcommand{\videosuppautorefname}{video}
+\newcommand{\srccodeautorefname}{source code}
+\newcommand{\FIGSUPP}[2][]{%
+ \ifstrequal{#1}{}{}{\autoref{fig:#1}---}\autoref{figsupp:#2}%
+}
+\newcommand{\FIGDATA}[2][]{%
+ \ifstrequal{#1}{}{}{\autoref{fig:#1}---}\autoref{figdata:#2}%
+}
+\newcommand{\TABLEDATA}[2][]{%
+ \ifstrequal{#1}{}{}{\autoref{tab:#1}---}\autoref{tabdata:#2}%
+}
+\newcommand{\VIDEODATA}[2][]{%
+ \ifstrequal{#1}{}{}{\autoref{video:#1}---}\autoref{viddata:#2}%
+}
+
+\newcommand{\FIGSRCCODE}[2][]{%
+ \ifstrequal{#1}{}{}{\autoref{fig:#1}---}\autoref{figsrccode:#2}%
+}
+\newcommand{\TABLESRCCODE}[2][]{%
+ \ifstrequal{#1}{}{}{\autoref{tab:#1}---}\autoref{tabsrccode:#2}%
+}
+\newcommand{\VIDEOSRCCODE}[2][]{%
+ \ifstrequal{#1}{}{}{\autoref{video:#1}---}\autoref{vidsrccode:#2}%
+}
+
+\newcommand{\VIDEOSUPP}[2][]{%
+ \ifstrequal{#1}{}{}{\autoref{fig:#1}---}\autoref{videosupp:#2}%
+}
+\newcommand{\FIG}[1]{\autoref{fig:#1}}
+\newcommand{\TABLE}[1]{\autoref{tab:#1}}
+\newcommand{\VIDEO}[1]{\autoref{video:#1}}
+\newcommand{\EQ}[1]{\autoref{eq:#1}}
+\newcommand{\BOX}[1]{\autoref{box:#1}}
+\let\oldautoref\autoref
+\renewcommand{\autoref}[1]{\emph{\textbf{\oldautoref{#1}}}}
+
+\endinput
diff --git a/paper/rsabsence.Rnw b/paper/rsabsence.Rnw
index ca02b0c297c919b504c4b700580753436042e3ec..203f9277497564531a4d90f0345310af1b6df0b2 100755
--- a/paper/rsabsence.Rnw
+++ b/paper/rsabsence.Rnw
@@ -1,65 +1,87 @@
-\documentclass[a4paper, 11pt]{article}
+\documentclass[9pt,lineno]{elife}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage[english]{babel}
-\usepackage{graphics}
-\usepackage[dvipsnames]{xcolor}
-\usepackage{amsmath, amssymb}
-\usepackage{doi} % automatic doi-links
-\usepackage[round]{natbib} % bibliography
-\usepackage{booktabs} % nicer tables
-\usepackage[title]{appendix} % better appendices
-\usepackage[onehalfspacing]{setspace} % more space
-\usepackage[labelfont=bf,font=small]{caption} % smaller captions
+\usepackage{doi}
+\usepackage{tikz} % to draw schematics
+\usetikzlibrary{decorations.pathreplacing,calligraphy} % for tikz curly braces
\usepackage{todonotes}
-%% margins
-\usepackage{geometry}
-\geometry{
- a4paper,
- total={170mm,257mm},
- left=25mm,
- right=25mm,
- top=30mm,
- bottom=25mm,
-}
+% \documentclass[a4paper, 11pt]{article}
+% \usepackage[T1]{fontenc}
+% \usepackage[utf8]{inputenc}
+% \usepackage[english]{babel}
+% \usepackage{graphics}
+% \usepackage[dvipsnames]{xcolor}
+% \usepackage{amsmath, amssymb}
+% \usepackage{doi} % automatic doi-links
+% \usepackage[round]{natbib} % bibliography
+% \usepackage{booktabs} % nicer tables
+% \usepackage[title]{appendix} % better appendices
+% \usepackage[onehalfspacing]{setspace} % more space
+% \usepackage[labelfont=bf,font=small]{caption} % smaller captions
+% \usepackage{tikz} % to draw schematics
+% \usetikzlibrary{decorations.pathreplacing,calligraphy} % for tikz curly braces
+% \usepackage{todonotes}
+% %% margins
+% \usepackage{geometry}
+% \geometry{
+% a4paper,
+% total={170mm,257mm},
+% left=25mm,
+% right=25mm,
+% top=30mm,
+% bottom=25mm,
+% }
+% \title{\vspace{-4em}
+% \textbf{Meta-research:\\
+% Replication of ``null results'' -- Absence of evidence or evidence of absence?}}
+% \author{{\bf Samuel Pawel\textsuperscript{*},
+% Rachel Heyard\textsuperscript{*},
+% Charlotte Micheloud,
+% Leonhard Held} \\
+% * contributed equally \\
+% Epidemiology, Biostatistics and Prevention Institute \\
+% Center for Reproducible Science \\
+% University of Zurich}
+% \date{\today} %don't forget to hard-code date when submitting to arXiv!
+% %% hyperref options
+% \usepackage{hyperref}
+% \hypersetup{
+% unicode=true,
+% bookmarksopen=true,
+% breaklinks=true,
+% colorlinks=true,
+% linkcolor=blue,
+% anchorcolor=black,
+% citecolor=blue,
+% urlcolor=black,
+% }
+
+\title{Meta-Research: Replication of ``null results'' -- Absence of evidence or
+ evidence of absence?}
+
+\author[1*\authfn{1}]{Samuel Pawel}
+\author[1\authfn{1}]{Rachel Heyard}
+\author[1]{Charlotte Micheloud}
+\author[1]{Leonhard Held}
+\affil[1]{Epidemiology, Biostatistics and Prevention Institute, Center for Reproducible Science, University of Zurich, Switzerland}
+
+\corr{samuel.pawel@uzh.ch}{SP}
+
+\contrib[\authfn{1}]{Contributed equally}
-\title{\vspace{-4em}
-\textbf{% Meta-research:\\
- Replication of ``null results'' -- Absence of evidence or evidence of absence?}}
-\author{{\bf Samuel Pawel\textsuperscript{*},
- Rachel Heyard\textsuperscript{*},
- Charlotte Micheloud,
- Leonhard Held} \\
- * contributed equally \\
- Epidemiology, Biostatistics and Prevention Institute \\
- Center for Reproducible Science \\
- University of Zurich}
-\date{\today} %don't forget to hard-code date when submitting to arXiv!
-
-%% hyperref options
-\usepackage{hyperref}
-\hypersetup{
- unicode=true,
- bookmarksopen=true,
- breaklinks=true,
- colorlinks=true,
- linkcolor=blue,
- anchorcolor=black,
- citecolor=blue,
- urlcolor=black,
-}
%% custom commands
\input{defs.tex}
\begin{document}
\maketitle
-%% Disclaimer that a preprint
-\vspace{-3em}
-\begin{center}
- {\color{red}This is a preprint which has not yet been peer reviewed.}
-\end{center}
+% %% Disclaimer that a preprint
+% \vspace{-3em}
+% \begin{center}
+% {\color{red}This is a preprint which has not yet been peer reviewed.}
+% \end{center}
<< "setup", include = FALSE >>=
## knitr options
@@ -122,41 +144,60 @@ BF01 <- function(estimate, se, null = 0, unitvar = 4) {
@
-%% Abstract
-%% -----------------------------------------------------------------------------
-\begin{center}
- \begin{minipage}{13cm} {\small
- \rule{\textwidth}{0.5pt} \\
- {\centering \textbf{Abstract} \\
- \textit{Absence of evidence is not evidence of absence} -- the title of
- the 1995 paper by Douglas Altman and Martin Bland has since become a
- mantra in the statistical and medical literature. Yet the
- misinterpretation of statistically non-significant results as evidence
- for the absence of an effect is still common and further complicated in
- the context of replication studies. In several large-scale replication
- projects, non-significant results in both the original and the
- replication study have been interpreted as a ``replication success''.
- Here we discuss the logical problems with this approach.
- Non-significance in both studies does not ensure that the studies
- provide evidence for the absence of an effect and
- % Because the null hypothesis of the statistical tests in both studies
- % is misaligned,
- ``replication success'' can virtually always be achieved if the sample
- sizes of the studies are small enough. In addition, the relevant error
- rates are not controlled. We show how methods, such as equivalence
- testing and Bayes factors, can be used to adequately quantify the
- evidence for the absence of an effect and how they can be applied in the
- replication setting. Using data from the Reproducibility Project: Cancer
- Biology we illustrate that most original and replication studies with
- ``null results'' are in fact inconclusive. We conclude that it is
- important to also replicate studies with statistically non-significant
- results, but that they should be designed, analyzed, and interpreted
- appropriately.
- } \\
- \rule{\textwidth}{0.5pt} \emph{Keywords}: Bayesian hypothesis testing,
- equivalence testing, meta-reasearch, null hypothesis, replication success}
- \end{minipage}
-\end{center}
+% %% Abstract
+% %% -----------------------------------------------------------------------------
+% \begin{center}
+% \begin{minipage}{13cm} {\small
+% \rule{\textwidth}{0.5pt} \\
+% {\centering \textbf{Abstract} \\
+% % \textit{Absence of evidence is not evidence of absence} -- the title of
+% % the 1995 paper by Douglas Altman and Martin Bland has since become a
+% % mantra in the statistical and medical literature. Yet the
+% % misinterpretation of statistically non-significant results as evidence
+% % for the absence of an effect is still common and further complicated in
+% % the context of replication studies.
+% In several large-scale replication
+% projects, non-significant results in both the original and the
+% replication study have been interpreted as a ``replication success''.
+% Here we discuss the logical problems with this approach.
+% Non-significance in both studies does not ensure that the studies
+% provide evidence for the absence of an effect and
+% % Because the null hypothesis of the statistical tests in both studies
+% % is misaligned,
+% ``replication success'' can virtually always be achieved if the sample
+% sizes of the studies are small enough. In addition, the relevant error
+% rates are not controlled. We show how methods, such as equivalence
+% testing and Bayes factors, can be used to adequately quantify the
+% evidence for the absence of an effect and how they can be applied in the
+% replication setting. Using data from the Reproducibility Project: Cancer
+% Biology we illustrate that most original and replication studies with
+% ``null results'' are in fact inconclusive. We conclude that it is
+% important to also replicate studies with statistically non-significant
+% results, but that they should be designed, analyzed, and interpreted
+% appropriately.
+% } \\
+% \rule{\textwidth}{0.5pt} \emph{Keywords}: Bayesian hypothesis testing,
+% equivalence testing, meta-research, null hypothesis, replication success}
+% \end{minipage}
+% \end{center}
+
+\begin{abstract}
+ In several large-scale replication projects, statistically non-significant
+ results in both the original and the replication study have been interpreted
+ as a ``replication success''. Here we discuss the logical problems with this
+ approach. Non-significance in both studies does not ensure that the studies
+ provide evidence for the absence of an effect and ``replication success'' can
+ virtually always be achieved if the sample sizes of the studies are small
+ enough. In addition, the relevant error rates are not controlled. We show how
+ methods, such as equivalence testing and Bayes factors, can be used to
+ adequately quantify the evidence for the absence of an effect and how they can
+ be applied in the replication setting. Using data from the Reproducibility
+ Project: Cancer Biology we illustrate that most original and replication
+ studies with ``null results'' are in fact inconclusive. We conclude that it is
+ important to also replicate studies with statistically non-significant
+ results, but that they should be designed, analyzed, and interpreted
+ appropriately.
+\end{abstract}
% definition from RPCP: null effects - the original authors interpreted their
% data as not showing evidence for a meaningful relationship or impact of an
@@ -165,21 +206,24 @@ BF01 <- function(estimate, se, null = 0, unitvar = 4) {
\section{Introduction}
-The misconception that a statistically non-significant result indicates evidence
-for the absence of an effect is unfortunately widespread \citep{Altman1995}.
-Such a ``null result'' -- typically characterized by a $p$-value of $p > 5\%$
-for the null hypothesis of an absent effect -- may also occur if an effect is
-actually present. For example, if the sample size of a study is chosen to detect
-an assumed effect with a power of 80\%, null results will incorrectly occur 20\%
-of the time when the assumed effect is actually present. Conversely, if the
-power of the study is lower, null results will occur more often. In general, the
-lower the power of a study, the greater the ambiguity of a null result. To put a
-null result in context, it is therefore critical to know whether the study was
+\textit{Absence of evidence is not evidence of absence} -- the title of the 1995
+paper by Douglas Altman and Martin Bland has since become a mantra in the
+statistical and medical literature \citep{Altman1995}. Yet, the misconception
+that a statistically non-significant result indicates evidence for the absence
+of an effect is unfortunately still widespread \citep{Makin2019}. Such a ``null
+result'' -- typically characterized by a $p$-value of $p > 0.05$ for the null
+hypothesis of an absent effect -- may also occur if an effect is actually
+present. For example, if the sample size of a study is chosen to detect an
+assumed effect with a power of 80\%, null results will incorrectly occur 20\% of
+the time when the assumed effect is actually present. Conversely, if the power
+of the study is lower, null results will occur more often. In general, the lower
+the power of a study, the greater the ambiguity of a null result. To put a null
+result in context, it is therefore critical to know whether the study was
adequately powered and under what assumed effect the power was calculated
\citep{Hoenig2001, Greenland2012}. However, if the goal of a study is to
explicitly quantify the evidence for the absence of an effect, more appropriate
-methods designed for this task, such as equivalence testing or Bayes factors,
-should be used from the outset.
+methods designed for this task, such as equivalence testing \citep{Wellek2010}
+or Bayes factors \citep{Kass1995}, should be used from the outset.
% two systematic reviews that I found which show that animal studies are very
% much underpowered on average \citep{Jennions2003,Carneiro2018}
@@ -218,12 +262,12 @@ sizes, such that the $p$-values are non-significant and the results are
inconclusive. This is because the null hypothesis under which the $p$-values are
computed is misaligned with the goal of inference, which is to quantify the
evidence for the absence of an effect. We will discuss methods that are better
-aligned with this inferential goal in Section~\ref{sec:methods}. Third, the
-criterion does not control the error of falsely claiming the absence of an
-effect at some predetermined rate. This is in contrast to the standard
+aligned with this inferential goal. % in Section~\ref{sec:methods}.
+Third, the criterion does not control the error of falsely claiming the absence
+of an effect at some predetermined rate. This is in contrast to the standard
replication success criterion of requiring significance from both studies
\citep[also known as the two-trials rule, see chapter 12.2.8 in][]{Senn2008},
-which ensures that the error of falsley claiming the presence of an effect is
+which ensures that the error of falsely claiming the presence of an effect is
controlled at a rate equal to the squared significance level (for example,
$5\% \times 5\% = 0.25\%$ for a $5\%$ significance level). The non-significance
criterion may be intended to complement the two-trials rule for null results,
@@ -328,7 +372,7 @@ rpcbNull <- rpcb %>%
Figure~\ref{fig:2examples} shows standardized mean difference effect estimates
with confidence intervals from two RPCB study pairs. Both are ``null results''
and meet the non-significance criterion for replication success (the two-sided
-$p$-values are greater than 5\% in both the original and the replication study),
+$p$-values are greater than 0.05 in both the original and the replication study),
but intuition would suggest that these two pairs are very much different.
\begin{figure}[ht]
<< "2-example-studies", fig.height = 3.25 >>=
@@ -404,8 +448,91 @@ zero (or negligibly small) effect. While these two examples show the qualitative
difference between absence of evidence and evidence of absence, we will now
discuss how the two can be quantitatively distinguished.
-\begin{figure}[!htb]
-<< "plot-null-findings-rpcb", fig.height = 8.25 >>=
+
+\section{Methods for asssessing replicability of null results}
+\label{sec:methods}
+There are both frequentist and Bayesian methods that can be used for assessing
+evidence for the absence of an effect. \citet{Anderson2016} provide an excellent
+summary of both approaches in the context of replication studies in psychology.
+We now briefly discuss two possible approaches -- frequentist equivalence
+testing and Bayesian hypothesis testing -- and their application to the RPCB
+data.
+
+
+
+\subsection{Equivalence testing}
+Equivalence testing was developed in the context of clinical trials to assess
+whether a new treatment -- typically cheaper or with fewer side effects than the
+established treatment -- is practically equivalent to the established treatment
+\citep{Westlake1972,Schuirmann1987}. The method can also be used to assess
+whether an effect is practically equivalent to the value of an absent effect,
+usually zero. Using equivalence testing as a remedy for non-significant results
+has been suggested by several authors \citep{Hauck1986, Campbell2018}. The main
+challenge is to specify the margin $\Delta > 0$ that defines an equivalence
+range $[-\Delta, +\Delta]$ in which an effect is considered as absent for
+practical purposes. The goal is then to reject
+the % composite %% maybe too technical?
+null hypothesis that the true effect is outside the equivalence range. This is
+in contrast to the usual null hypothesis of a superiority test which states that
+the effect is zero or smaller than zero, see Figure~\ref{fig:hypotheses} for an
+illustration.
+
+\begin{figure}
+ \begin{center}
+ \begin{tikzpicture}[ultra thick]
+ \draw[stealth-stealth] (0,0) -- (6,0);
+ \node[text width=4.5cm, align=center] at (3,-1) {Effect size};
+ \draw (2,0.2) -- (2,-0.2) node[below]{$-\Delta$};
+ \draw (3,0.2) -- (3,-0.2) node[below]{$0$};
+ \draw (4,0.2) -- (4,-0.2) node[below]{$+\Delta$};
+
+ \node[text width=5cm, align=left] at (9.5,1.5) {Equivalence test};
+ \draw [draw={red},decorate,decoration={brace,amplitude=5pt}]
+ (2.05,1) -- (3.95,1) node[midway,yshift=1.5em]{\textcolor{red}{$H_1$}};
+ \draw [draw={blue},decorate,decoration={brace,amplitude=5pt,aspect=0.6}]
+ (0,1) -- (1.95,1) node[pos=0.6,yshift=1.5em]{\textcolor{blue}{$H_0$}};
+ \draw [draw={blue},decorate,decoration={brace,amplitude=5pt,aspect=0.4}]
+ (4.05,1) -- (6,1) node[pos=0.4,yshift=1.5em]{\textcolor{blue}{$H_0$}};
+
+ \node[text width=5cm, align=left] at (9.5,3.5) {Superiority test (two-sided)};
+ \draw [decorate,decoration={brace,amplitude=5pt}]
+ (3,3) -- (3,3) node[midway,yshift=1.5em]{\textcolor{blue}{$H_0$}};
+ \draw[blue] (3,2.8) -- (3,3.2);
+ \draw [draw={red},decorate,decoration={brace,amplitude=5pt,aspect=0.6}]
+ (0,3) -- (2.95,3) node[pos=0.6,yshift=1.5em]{\textcolor{red}{$H_1$}};
+ \draw [draw={red},decorate,decoration={brace,amplitude=5pt,aspect=0.4}]
+ (3.05,3) -- (6,3) node[pos=0.4,yshift=1.5em]{\textcolor{red}{$H_1$}};
+
+ \node[text width=5cm, align=left] at (9.5,5.5) {Superiority test (one-sided)};
+ \draw [draw={red},decorate,decoration={brace,amplitude=5pt,aspect=0.4}]
+ (3.05,5) -- (6,5) node[pos=0.4,yshift=1.5em]{\textcolor{red}{$H_1$}};
+ \draw [draw={blue},decorate,decoration={brace,amplitude=5pt,aspect=0.6}]
+ (0,5) -- (3,5) node[pos=0.6,yshift=1.5em]{\textcolor{blue}{$H_0$}};
+
+ \draw [dashed] (2,0) -- (2,1);
+ \draw [dashed] (4,0) -- (4,1);
+ \draw [dashed] (3,0) -- (3,1);
+ \draw [dashed] (3,1.9) -- (3,2.7);
+ \draw [dashed] (3,3.9) -- (3,5);
+ \end{tikzpicture}
+ \end{center}
+ \caption{Null hypothesis ($H_0$) and alternative hypothesis ($H_1$) for
+ different study designs with equivalence margin $\Delta$.}
+ \label{fig:hypotheses}
+\end{figure}
+
+To ensure that the null hypothesis is falsely rejected at most
+$\alpha \times 100\%$ of the time, one either rejects it if the
+$(1-2\alpha)\times 100\%$ confidence interval for the effect is contained within
+the equivalence range (for example, a 90\% confidence interval for
+$\alpha = 5\%$), or if two one-sided tests (TOST) for the effect being
+smaller/greater than $+\Delta$ and $-\Delta$ are significant at level $\alpha$,
+respectively. A quantitative measure of evidence for the absence of an effect is
+then given by the maximum of the two one-sided $p$-values (the TOST $p$-value).
+
+\begin{figure}
+ \begin{fullwidth}
+<< "plot-null-findings-rpcb", fig.height = 8.25, fig.width = "0.95\\linewidth" >>=
## compute TOST p-values
margin <- 1
conflevel <- 0.9
@@ -471,7 +598,7 @@ ggplot(data = rpcbNull) +
@
\caption{Standardized mean difference (SMD) effect estimates with
\Sexpr{round(conflevel*100, 2)}\% confidence interval for the ``null results''
- (those with original two-sided $p$-value $p > 5\%$) and their replication
+ (those with original two-sided $p$-value $p > 0.05$) and their replication
studies from the Reproducibility Project: Cancer Biology
\citep{Errington2021}. The identifier above each plot indicates (Original
paper number, Experiment number, Effect number, Internal replication number).
@@ -485,44 +612,13 @@ ggplot(data = rpcbNull) +
alternative $H_{1} \colon \text{SMD} \neq 0$ with normal unit-information
prior assigned to the SMD under $H_{1}$.}
\label{fig:nullfindings}
+\end{fullwidth}
\end{figure}
-
-\section{Methods for asssessing replicability of null results}
-\label{sec:methods}
-There are both frequentist and Bayesian methods that can be used for assessing
-evidence for the absence of an effect. \citet{Anderson2016} provide an excellent
-summary of both approaches in the context of replication studies in psychology.
-We now briefly discuss two possible approaches -- frequentist equivalence
-testing and Bayesian hypothesis testing -- and their application to the RPCB
-data.
-
-
-\subsection{Equivalence testing}
-Equivalence testing was developed in the context of clinical trials to assess
-whether a new treatment -- typically cheaper or with fewer side effects than the
-established treatment -- is practically equivalent to the established treatment
-\citep{Westlake1972,Schuirmann1987}. The method can also be used to assess
-whether an effect is practically equivalent to the value of an absent effect,
-usually zero. Using equivalence testing as a remedy for non-significant results
-has been suggested by several authors \citep{Hauck1986, Campbell2018}. The main
-challenge is to specify the margin $\Delta > 0$ that defines an equivalence
-range $[-\Delta, +\Delta]$ in which an effect is considered as absent for
-practical purposes. The goal is then to reject the composite null hypothesis
-that the true effect is outside the equivalence range. To ensure that the null
-hypothesis is falsely rejected at most $\alpha \times 100\%$ of the time, one
-either rejects it if the $(1-2\alpha)\times 100\%$ confidence interval for the
-effect is contained within the equivalence range (for example, a 90\% confidence
-interval for $\alpha = 5\%$), or if two one-sided tests (TOST) for the effect
-being smaller/greater than $+\Delta$ and $-\Delta$ are significant at level
-$\alpha$, respectively. A quantitative measure of evidence for the absence of an
-effect is then given by the maximum of the two one-sided $p$-values (the TOST
-$p$-value).
-
Returning to the RPCB data, Figure~\ref{fig:nullfindings} shows the standarized
mean difference effect estimates with \Sexpr{round(conflevel*100, 2)}\%
confidence intervals for the 20 study pairs with quantitative null results in
-the original study ($p > 5\%$). The dotted red lines represent an equivalence
+the original study ($p > 0.05$). The dotted red lines represent an equivalence
range for the margin $\Delta = \Sexpr{margin}$, for which the shown TOST
$p$-values are computed. This margin is rather lax compared to the margins
typically used in clinical research; we chose it primarily for illustrative
@@ -534,11 +630,10 @@ one of them being the previously discussed example from \citet{Goetz2011} -- are
able to establish equivalence at the 5\% level in the sense that both the
original and the replication 90\% confidence interval fall within the
equivalence range (or equivalently that their TOST $p$-values are smaller than
-$5\%$). For the remaining 16 studies -- for instance, the previously discussed
+$0.05$). For the remaining 16 studies -- for instance, the previously discussed
example from \citet{Dawson2011} -- the situation remains inconclusive and there
is neither evidence for the absence nor the presence of the effect.
-
\subsection{Bayesian hypothesis testing}
The distinction between absence of evidence and evidence of absence is naturally
built into the Bayesian approach to hypothesis testing. A central measure of
@@ -561,10 +656,10 @@ the presence of the effect (\mbox{$\BF_{01} < 1$}), whereas a Bayes factor not
much different from one indicates absence of evidence for either hypothesis
(\mbox{$\BF_{01} \approx 1$}).
-When the observed data are dichotomized into positive (\mbox{$p < 5\%$}) or null
-results (\mbox{$p > 5\%$}), the Bayes factor based on a null result is the
-probability of observing \mbox{$p > 5\%$} when the effect is indeed absent
-(which is $95\%$) divided by the probability of observing $p > 5\%$ when the
+When the observed data are dichotomized into positive (\mbox{$p < 0.05$}) or null
+results (\mbox{$p > 0.05$}), the Bayes factor based on a null result is the
+probability of observing \mbox{$p > 0.05$} when the effect is indeed absent
+(which is $95\%$) divided by the probability of observing $p > 0.05$ when the
effect is indeed present (which is one minus the power of the study). For
example, if the power is 90\%, we have
\mbox{$\BF_{01} = 95\%/10\% = \Sexpr{round(0.95/0.1, 2)}$} indicating almost ten
@@ -629,7 +724,7 @@ conclusion -- most RPCB null results are highly ambiguous.
\section{Conclusions}
We showed that in most of the RPCB studies with ``null results'' (those with
-$p > 5\%$), neither the original nor the replication study provided conclusive
+$p > 0.05$), neither the original nor the replication study provided conclusive
evidence for the presence or absence of an effect. It seems logically
questionable to declare an inconclusive replication of an inconclusive original
study as a replication success. While it is important to replicate original
@@ -713,7 +808,7 @@ language R version \Sexpr{paste(version$major, version$minor, sep = ".")}
preparation, dynamic reporting, and formatting, respectively.
-\bibliographystyle{apalikedoiurl}
+% \bibliographystyle{apalikedoiurl}
\bibliography{bibliography}
<< >>=
diff --git a/paper/vancouver-elife.bst b/paper/vancouver-elife.bst
new file mode 100644
index 0000000000000000000000000000000000000000..233525827f0d6e48681373ba7f22c0db1846116a
--- /dev/null
+++ b/paper/vancouver-elife.bst
@@ -0,0 +1,1935 @@
+%% vancouver-elife.bst (v 1.2, 6 July 2017) is modified
+%% by LianTze Lim (Overleaf)
+%% from the
+%% natbib-compatible BibTeX bibliography style `vancouver-authoryear' at https://github.com/gbhutani/vancouver_authoryear_bibstyle/
+%%
+%% Use
+%%
+%% \usepackage{natbib}
+%% \bibliographystyle{vancouver-compatible}
+%%
+%% and cite references with (e.g.)
+%%
+%% \cite{smith77} % to get a "[1]" in the text
+%% \citep{smith77} % to get a "[1]" in the text
+%% \citet{smith77} % to get a "Smith [1]" in the text
+%% \citeauthor{smith77} % to get a "Smith" in the text
+%%
+%% The changes below are inspired by similar changes made to
+%% splncs03.bst by Maurizio "Titto" Patrignani of
+%% Dipartimento di Informatica e Automazione Universita' Roma Tre.
+%% Unfortunately, splncs03.bst was not compatible with natbib (because it
+%% was not built with author-year capability).
+%%
+%% This is derived from `splncsnat.bst',
+%---------------------------------------------------------------------
+
+ENTRY
+ { address
+ assignee % for patents
+ author
+ booktitle % for articles in books
+ chapter % for incollection, esp. internet documents
+ cartographer % for maps
+ day
+ edition
+ editor
+ eid
+ howpublished
+ institution % for technical reports
+ inventor % for patents
+ journal
+ key
+ month
+ note
+ number
+ organization
+ pages
+ part
+ publisher
+ school
+ series
+ title
+ type
+ url
+ doi
+ volume
+ word
+ year
+ }
+ {}
+ { label extra.label sort.label short.list }
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+ #1 'mid.sentence :=
+ #2 'after.sentence :=
+ #3 'after.block :=
+}
+%% Declaration of string variables
+STRINGS { s t}
+FUNCTION {output.nonnull}
+{ 's :=
+ output.state mid.sentence =
+ { ", " * write$ }
+ { output.state after.block =
+ { add.period$ write$
+ newline$
+ "\newblock " write$
+ }
+ { output.state before.all =
+ 'write$
+ { " " * write$ }
+ if$
+ }
+ if$
+ mid.sentence 'output.state :=
+ }
+ if$
+ s
+}
+FUNCTION {output}
+{ duplicate$ empty$
+ 'pop$
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.check}
+{ 't :=
+ duplicate$ empty$
+ { pop$ "empty " t * " in " * cite$ * warning$ }
+ 'output.nonnull
+ if$
+}
+
+%FUNCTION {fin.entry}
+%{ duplicate$ empty$
+% 'pop$
+% 'write$
+% if$
+% newline$
+%}
+%
+FUNCTION {fin.entry}
+{ add.period$
+ write$
+ newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+ 'skip$
+ { after.block 'output.state := }
+ if$
+}
+
+FUNCTION {new.sentence}
+{ output.state after.block =
+ 'skip$
+ { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+ if$
+ }
+ if$
+}
+
+FUNCTION {add.blank}
+{ " " * before.all 'output.state :=
+}
+
+FUNCTION {no.blank.or.punct}
+{ "" * before.all 'output.state :=
+}
+
+FUNCTION {add.semicolon}
+{
+ ";" *
+% no.blank.or.punct
+}
+
+FUNCTION {date.block}
+{
+ "." *
+ no.blank.or.punct
+}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% LOGICAL `NOT', `AND', AND `OR' %
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Logical 'not':
+% If the first element on the stack is A then this function
+% does the following:
+% push { #0 }
+% push { #1 }
+% So now the first 3 elements of the stack are
+% { #1 } { #0 } A
+% The first 3 are popped and subjected to 'if':
+% If A > 0 then { #0 } is executed, else { #1 } is executed:
+% if A > 0
+% then 0
+% else 1
+% So consider integers as logicals, where 1 = true and 0 = false,
+% then this does
+% (if A then false else true)
+% which is a logical 'not'.
+
+FUNCTION {not}
+{ { #0 }
+ { #1 }
+ if$
+}
+FUNCTION {and}
+{ 'skip$
+ { pop$ #0 }
+ if$
+}
+FUNCTION {or}
+{ { pop$ #1 }
+ 'skip$
+ if$
+}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% GENERAL PURPOSE FUNCTIONS FOR FORMATTING %
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% issues warning if field is empty
+% call with
+% "field" field warning.if.empty
+% Note that the first field must be between quotes
+% because it is the fieldname for use in the warning message.
+%
+
+FUNCTION {warning.if.empty}
+{ empty$
+ { "No " swap$ * " in " * cite$ * warning$ }
+ { pop$ }
+ if$
+}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ %
+ % encloses string in pre- and postfix string
+ % call with
+ % prefix postfix S enclose.check
+ % delivers empty string if S empty
+ %
+FUNCTION {enclose.check}
+{ duplicate$ empty$
+ { pop$ pop$ pop$
+ ""
+ }
+ { swap$ * * }
+ if$
+}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%
+% emphasizes top of stack
+% call with
+% string" emphasize.check
+%
+
+FUNCTION {emphasize.check}
+{ "\Bem{" swap$
+ "}" swap$
+ enclose.check
+}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ %
+ % brackets top of stack
+ % call with
+ % "string" bracket.check
+ %
+FUNCTION {bracket.check}
+{ "[" swap$
+ "]" swap$
+ enclose.check
+}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ %
+ % parenthesizes top of stack
+ % call with
+ % "string" parenthesize
+ %
+FUNCTION {parenthesize.check}
+{ "(" swap$
+ ")" swap$
+ enclose.check
+}
+
+STRINGS {z}
+
+FUNCTION {remove.dots}
+{ 'z := % expects string on top of the stack, pops the string and assigns it to variable z
+ "" % push empty string
+ { z empty$ not } % returns 0 if variable z is empty
+ { z #1 #1 substring$ % push the first character of variable z
+ z #2 global.max$ substring$ 'z := % assigns the 2nd to last character of variable z to variable z
+ duplicate$ "\" = % pushes 1 if the last character is "\", otherwise 0
+ { * % concatenates the last 2 literals
+ z #1 #1 substring$ % push the first character of variable z
+ z #2 global.max$ substring$ 'z := % assigns the 2nd to last character of variable z to variable z
+ * % concatenates the last 2 literals, i.e. every character, even a dot, following a "\" will be printed
+ }
+ { duplicate$ "." = % pushes 1 if the last character is ".", otherwise 0
+ 'pop$ % pushes the pop$ function
+ { * } % concatenates the last 2 literals
+ if$ % pops the last character if it is a dot, otherwise concatenates it with the string on top of the stack
+ }
+ if$
+ }
+ while$
+}
+
+INTEGERS {l}
+FUNCTION{string.length}
+{
+ #1 'l :=
+ { duplicate$ duplicate$ #1 l substring$ = not }
+ { l #1 + 'l := }
+ while$
+ pop$ l
+}
+
+STRINGS {replace find text}
+INTEGERS {find_length}
+FUNCTION {find.replace}
+{
+ 'replace :=
+ 'find :=
+ 'text :=
+ find string.length 'find_length :=
+ ""
+ { text empty$ not }
+ { text #1 find_length substring$ find =
+ {
+ replace *
+ text #1 find_length + global.max$ substring$ 'text :=
+ }
+ { text #1 #1 substring$ *
+ text #2 global.max$ substring$ 'text :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {new.block.checka}
+{ empty$
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.block.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.sentence.checka}
+{ empty$
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {new.sentence.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+ { pop$ "" }
+ 'skip$
+ if$
+}
+
+FUNCTION {emphasize}
+{ skip$ }
+
+FUNCTION {tie.or.space.prefix}
+{ duplicate$ text.length$ #3 <
+ { "~" }
+ { " " }
+ if$
+ swap$
+}
+
+FUNCTION {capitalize}
+{ "u" change.case$ "t" change.case$ }
+
+FUNCTION {space.word}
+{ " " swap$ * " " * }
+
+ % Here are the language-specific definitions for explicit words.
+ % Each function has a name bbl.xxx where xxx is the English word.
+ % The language selected here is ENGLISH
+
+FUNCTION {bbl.and}
+{ "and"}
+
+FUNCTION {bbl.etal}
+{ "et~al." }
+
+FUNCTION {bbl.editors}
+{ "editors" }
+
+FUNCTION {bbl.editor}
+{ "editor" }
+
+FUNCTION {bbl.cartographers}
+{ "cartographers" }
+
+FUNCTION {bbl.cartographer}
+{ "cartographer" }
+
+FUNCTION {bbl.inventors}
+{ "inventors" }
+
+FUNCTION {bbl.inventor}
+{ "inventor" }
+
+FUNCTION {bbl.assignees}
+{ "assignees" }
+
+FUNCTION {bbl.assignee}
+{ "assignee" }
+
+FUNCTION {bbl.edby}
+{ "edited by" }
+
+FUNCTION {bbl.edition}
+{ "ed." }
+
+FUNCTION {bbl.volume}
+{ "vol." }
+
+FUNCTION {bbl.of}
+{ "of" }
+
+FUNCTION {bbl.number}
+{ "no." }
+
+FUNCTION {bbl.nr}
+{ "no." }
+
+FUNCTION {bbl.in}
+{ "in" }
+
+FUNCTION {bbl.pages}
+{ "p." }
+
+FUNCTION {bbl.page}
+{ "p." }
+
+FUNCTION {bbl.chapter}
+{ "chap." }
+
+FUNCTION {bbl.techrep}
+{ "Tech. Rep." }
+
+FUNCTION {bbl.mthesis}
+{ "Master's thesis" }
+
+FUNCTION {bbl.phdthesis}
+{ "Ph.D. thesis" }
+
+FUNCTION {bbl.first}
+{ "1st" }
+
+FUNCTION {bbl.second}
+{ "2nd" }
+
+FUNCTION {bbl.third}
+{ "3rd" }
+
+FUNCTION {bbl.fourth}
+{ "4th" }
+
+FUNCTION {bbl.fifth}
+{ "5th" }
+
+FUNCTION {bbl.st}
+{ "st" }
+
+FUNCTION {bbl.nd}
+{ "nd" }
+
+FUNCTION {bbl.rd}
+{ "rd" }
+
+FUNCTION {bbl.th}
+{ "th" }
+
+MACRO {jan} {"Jan."}
+
+MACRO {feb} {"Feb."}
+
+MACRO {mar} {"Mar."}
+
+MACRO {apr} {"Apr."}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"Jun."}
+
+MACRO {jul} {"Jul."}
+
+MACRO {aug} {"Aug."}
+
+MACRO {sep} {"Sep."}
+
+MACRO {oct} {"Oct."}
+
+MACRO {nov} {"Nov."}
+
+MACRO {dec} {"Dec."}
+
+MACRO {acmcs} {"ACM Comput. Surv."}
+
+MACRO {acta} {"Acta Inf."}
+
+MACRO {cacm} {"Commun. ACM"}
+
+MACRO {ibmjrd} {"IBM J. Res. Dev."}
+
+MACRO {ibmsj} {"IBM Syst.~J."}
+
+MACRO {ieeese} {"IEEE Trans. Software Eng."}
+
+MACRO {ieeetc} {"IEEE Trans. Comput."}
+
+MACRO {ieeetcad}
+ {"IEEE Trans. Comput. Aid. Des."}
+
+MACRO {ipl} {"Inf. Process. Lett."}
+
+MACRO {jacm} {"J.~ACM"}
+
+MACRO {jcss} {"J.~Comput. Syst. Sci."}
+
+MACRO {scp} {"Sci. Comput. Program."}
+
+MACRO {sicomp} {"SIAM J. Comput."}
+
+MACRO {tocs} {"ACM Trans. Comput. Syst."}
+
+MACRO {tods} {"ACM Trans. Database Syst."}
+
+MACRO {tog} {"ACM Trans. Graphic."}
+
+MACRO {toms} {"ACM Trans. Math. Software"}
+
+MACRO {toois} {"ACM Trans. Office Inf. Syst."}
+
+MACRO {toplas} {"ACM Trans. Progr. Lang. Syst."}
+
+MACRO {tcs} {"Theor. Comput. Sci."}
+
+FUNCTION {eng.ord}
+{ duplicate$ "1" swap$ *
+ #-2 #1 substring$ "1" =
+ { bbl.th * }
+ { duplicate$ #-1 #1 substring$
+ duplicate$ "1" =
+ { pop$ bbl.st * }
+ { duplicate$ "2" =
+ { pop$ bbl.nd * }
+ { "3" =
+ { bbl.rd * }
+ { bbl.th * }
+ if$
+ }
+ if$
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {bibinfo.check}
+{ swap$
+ duplicate$ missing$
+ {
+ pop$ pop$
+ ""
+ }
+ { duplicate$ empty$
+ {
+ swap$ pop$
+ }
+ { swap$
+ pop$
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {bibinfo.warn}
+{ swap$
+ duplicate$ missing$
+ {
+ swap$ "missing " swap$ * " in " * cite$ * warning$ pop$
+ ""
+ }
+ { duplicate$ empty$
+ {
+ swap$ "empty " swap$ * " in " * cite$ * warning$
+ }
+ { swap$
+ pop$
+ }
+ if$
+ }
+ if$
+}
+INTEGERS { nameptr namesleft numnames }
+
+
+STRINGS { bibinfo}
+
+FUNCTION {format.names}
+{ 'bibinfo :=
+ duplicate$ empty$ 'skip$ {
+ "." ". " find.replace 's :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv~}{ll}{ f{}}{ jj}"
+ format.name$
+ remove.dots
+ bibinfo bibinfo.check
+ 't :=
+ nameptr #1 >
+ {
+ nameptr #20
+ #1 + =
+ numnames #20
+ > and
+ { "others" 't :=
+ #1 'namesleft := }
+ 'skip$
+ if$
+ namesleft #1 >
+ { ", " * t * }
+ {
+ "," *
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ { " " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+ } if$
+}
+
+FUNCTION {format.names.auth}
+{ 'bibinfo :=
+ duplicate$ empty$ 'skip$ {
+ "." ". " find.replace 's :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{vv~}{ll}{ f{}}{ jj}"
+ %% Highlight first author for eLife
+ nameptr #1 =
+ {format.name$ "\textbf{\color{eLifeMediumGrey} " swap$ * "}" * }
+ {format.name$ }
+ if$
+ remove.dots
+ bibinfo bibinfo.check
+ 't :=
+ nameptr #1 >
+ {
+ nameptr #20
+ #1 + =
+ numnames #20
+ > and
+ { "others" 't :=
+ #1 'namesleft := }
+ 'skip$
+ if$
+ namesleft #1 >
+ { ", " * t * }
+ {
+ "," *
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ { " " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+ } if$
+}
+
+FUNCTION {format.names.org.original}
+{ 'bibinfo :=
+ duplicate$ empty$ 'skip$ {
+ 's :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{ff~}{vv~}{ll}"
+ format.name$
+ bibinfo bibinfo.check
+ 't :=
+ nameptr #1 >
+ {
+ namesleft #1 >
+ { "; " * t * }
+ {
+ ";" *
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ { " " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+ } if$
+}
+
+FUNCTION {format.names.org}
+{ 'bibinfo :=
+ duplicate$ empty$ 'skip$ {
+ 's :=
+ "" 't :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{ff~}{vv~}{ll}"
+ nameptr #1 =
+ {format.name$ "\textbf{\color{eLifeMediumGrey} " swap$ * "}" * }
+ {format.name$}
+ if$
+ bibinfo bibinfo.check
+ 't :=
+ nameptr #1 >
+ {
+ namesleft #1 >
+ { "; " * t * }
+ {
+ ";" *
+ s nameptr "{ll}" format.name$ duplicate$ "others" =
+ { 't := }
+ { pop$ }
+ if$
+ t "others" =
+ {
+ " " * bbl.etal *
+ }
+ { " " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+ } if$
+}
+
+FUNCTION {format.names.ed}
+{
+ format.names
+}
+FUNCTION {format.key}
+{ empty$
+ { key field.or.null }
+ { "" }
+ if$
+}
+
+FUNCTION {format.authors}
+{
+ author "author" format.names.auth
+ %%"." " " "author" find.replace format.names
+}
+
+FUNCTION {format.organizations}
+{ organization "organization" format.names.org
+}
+
+FUNCTION {get.bbl.editor}
+{ editor num.names$ #1 > 'bbl.editors 'bbl.editor if$ }
+
+FUNCTION {get.bbl.cartographer}
+{ cartographer num.names$ #1 > 'bbl.cartographers 'bbl.cartographer if$ }
+
+FUNCTION {get.bbl.inventor}
+{ inventor num.names$ #1 > 'bbl.inventors 'bbl.inventor if$ }
+
+FUNCTION {get.bbl.assignee}
+{ assignee num.names$ #1 > 'bbl.assignees 'bbl.assignee if$ }
+
+FUNCTION {format.editors}
+{ editor "editor" format.names duplicate$ empty$ 'skip$
+ {
+ "," *
+ " " *
+ get.bbl.editor
+ *
+ }
+ if$
+}
+
+FUNCTION {format.assignees}
+{ assignee "assignee" format.names.org duplicate$ empty$ 'skip$
+ {
+ "," *
+ " " *
+ get.bbl.assignee
+ *
+ }
+ if$
+}
+
+FUNCTION {format.cartographers}
+{ cartographer "cartographer" format.names duplicate$ empty$ 'skip$
+ {
+ "," *
+ " " *
+ get.bbl.cartographer
+ *
+ }
+ if$
+}
+
+FUNCTION {format.inventors}
+{ inventor "inventor" format.names duplicate$ empty$ 'skip$
+ {
+ "," *
+ " " *
+ get.bbl.inventor
+ *
+ }
+ if$
+}
+
+FUNCTION {format.note}
+{
+ url empty$
+ 'skip$
+ { "\urlprefix\url{" url * "}" * output }
+ if$
+ doi empty$
+ 'skip$
+ { "\href{" doi * "}{\doiprefix \detokenize{" * doi * "}}" * output }
+ if$
+ note empty$
+ { "" }
+ { note #1 #1 substring$
+ duplicate$ "{" =
+ 'skip$
+ { output.state mid.sentence =
+ { "l" }
+ { "u" }
+ if$
+ change.case$
+ }
+ if$
+ note #2 global.max$ substring$ * "note" bibinfo.check
+ }
+ if$
+}
+
+FUNCTION {format.title}
+{ title
+%%duplicate$ empty$ 'skip$
+%% { "t" change.case$ }
+%%if$
+ "title" bibinfo.check
+}
+
+
+FUNCTION {author.editor.key.full}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor }
+ if$
+ }
+ { author }
+ if$
+}
+
+FUNCTION {author.key.full}
+{ author empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { author }
+ if$
+}
+
+FUNCTION {editor.key.full}
+{ editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor }
+ if$
+}
+
+FUNCTION {make.full.names}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.full
+ { type$ "proceedings" =
+ 'editor.key.full
+ 'author.key.full
+ if$
+ }
+ if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+ "\bibitem[{" write$
+ label write$
+ ")" make.full.names duplicate$ short.list =
+ { pop$ }
+ { * }
+ if$
+ "}]{" * write$
+ cite$ write$
+ "}" write$
+ newline$
+ ""
+ before.all 'output.state :=
+}
+
+FUNCTION {n.dashify}
+{
+ 't :=
+ ""
+ { t empty$ not }
+ { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+ { "--" *
+ t #2 global.max$ substring$ 't :=
+ }
+ { { t #1 #1 substring$ "-" = }
+ { "-" *
+ t #2 global.max$ substring$ 't :=
+ }
+ while$
+ }
+ if$
+ }
+ { t #1 #1 substring$ *
+ t #2 global.max$ substring$ 't :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {word.in}
+{ bbl.in capitalize
+ ":" *
+ " " * }
+
+FUNCTION {format.journal.date}
+{
+ month "month" bibinfo.check
+ duplicate$ empty$
+ year "year" bibinfo.check duplicate$ empty$
+ {
+ swap$ 'skip$
+ { "there's a month but no year in " cite$ * warning$ }
+ if$
+ *
+ }
+ { swap$ 'skip$
+ {
+ " " * swap$
+ }
+ if$
+ *
+ remove.dots
+ }
+ if$
+ duplicate$ empty$
+ 'skip$
+ {
+ before.all 'output.state :=
+ after.sentence 'output.state :=
+ }
+ if$
+}
+
+FUNCTION {format.date}
+{
+ no.blank.or.punct
+ ";"
+ duplicate$ empty$
+ year "year" bibinfo.check duplicate$ empty$
+ { swap$ 'skip$
+ { "there's a month but no year in " cite$ * warning$ }
+ if$
+ *
+ }
+ { swap$ 'skip$
+ {
+ swap$
+ " " * swap$
+ }
+ if$
+ *
+ }
+ if$
+}
+
+FUNCTION {format.btitle}
+{ title "title" bibinfo.check
+ duplicate$ empty$ 'skip$
+ {
+ }
+ if$
+}
+
+FUNCTION {either.or.check}
+{ empty$
+ 'pop$
+ { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+ if$
+}
+
+FUNCTION {format.bvolume}
+{ volume empty$
+ { "" }
+ { bbl.volume volume tie.or.space.prefix
+ "volume" bibinfo.check * *
+ series "series" bibinfo.check
+ duplicate$ empty$ 'pop$
+ { swap$ bbl.of space.word * swap$
+ emphasize * }
+ if$
+ "volume and number" number either.or.check
+ }
+ if$
+}
+
+FUNCTION {format.number.series}
+{ volume empty$
+ { number empty$
+ { series field.or.null }
+ { output.state mid.sentence =
+ { bbl.number }
+ { bbl.number capitalize }
+ if$
+ number tie.or.space.prefix "number" bibinfo.check * *
+ series empty$
+ { "there's a number but no series in " cite$ * warning$ }
+ { bbl.in space.word *
+ series "series" bibinfo.check *
+ }
+ if$
+ }
+ if$
+ }
+ { "" }
+ if$
+}
+
+FUNCTION {format.edition}
+{ edition duplicate$ empty$ 'skip$
+ {
+% convert.edition
+ output.state mid.sentence =
+ { "l" }
+ { "t" }
+ if$ change.case$
+ "edition" bibinfo.check
+ " " * bbl.edition *
+ }
+ if$
+}
+INTEGERS { multiresult }
+FUNCTION {multi.page.check}
+{ 't :=
+ #0 'multiresult :=
+ { multiresult not
+ t empty$ not
+ and
+ }
+ { t #1 #1 substring$
+ duplicate$ "-" =
+ swap$ duplicate$ "," =
+ swap$ "+" =
+ or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+ if$
+ }
+ while$
+ multiresult
+}
+
+FUNCTION {format.pages}
+{ pages duplicate$ empty$ 'skip$
+ { duplicate$ multi.page.check
+ {
+ bbl.pages swap$
+ n.dashify
+ }
+ {
+ bbl.page swap$
+ }
+ if$
+ tie.or.space.prefix
+ "pages" bibinfo.check
+ * *
+ }
+ if$
+}
+
+FUNCTION {format.journal.pages}
+{ pages duplicate$ empty$ 'pop$
+ { swap$ duplicate$ empty$
+ { pop$ pop$ format.pages }
+ {
+ ":" *
+ swap$
+ n.dashify
+ "pages" bibinfo.check
+ *
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.vol.num}
+{ volume field.or.null
+ duplicate$ empty$ 'skip$
+ {
+ "volume" bibinfo.check
+ }
+ if$
+ number "number" bibinfo.check duplicate$ empty$ 'skip$
+ {
+ swap$ duplicate$ empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ swap$
+ "(" swap$ * ")" *
+ }
+ if$ *
+}
+
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+ duplicate$ empty$ 'skip$
+ {
+ "volume" bibinfo.check
+ }
+ if$
+ number "number" bibinfo.check duplicate$ empty$ 'skip$
+ {
+ swap$ duplicate$ empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ swap$
+ "(" swap$ * ")" *
+ }
+ if$ *
+ format.journal.pages
+}
+
+FUNCTION {format.volume}
+{ volume "\textbf{" swap$ * "}" * }
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+ 'format.pages
+ { type empty$
+ { bbl.chapter }
+ { type "l" change.case$
+ "type" bibinfo.check
+ }
+ if$
+ chapter tie.or.space.prefix
+ "chapter" bibinfo.check
+ * *
+ pages empty$
+ 'skip$
+ { ", " * format.pages * }
+ if$
+ }
+ if$
+ }
+
+ FUNCTION {format.booktitle}
+ {
+ booktitle "\emph{" swap$ * "}" * "booktitle" bibinfo.check
+ }
+
+ FUNCTION {format.journal}
+ {
+ journal "\emph{" swap$ * "}" * "journal" bibinfo.check
+ }
+
+
+ FUNCTION {format.in.ed.booktitle}
+ { format.booktitle duplicate$ empty$ 'skip$
+ {
+ editor "editor" format.names.ed duplicate$ empty$ 'pop$
+ {
+ "," *
+ " " *
+ get.bbl.editor
+ ". " *
+ * swap$
+ * }
+ if$
+ word.in swap$ *
+ }
+ if$
+ }
+
+ FUNCTION {format.in.ed.title}
+ { format.title duplicate$ empty$ 'skip$
+ {
+ editor "editor" format.names.ed duplicate$ empty$ 'pop$
+ {
+ "," *
+ " " *
+ get.bbl.editor
+ ". " *
+ * swap$
+ * }
+ if$
+ word.in swap$ *
+ }
+ if$
+ }
+
+ FUNCTION {empty.misc.check}
+ { author empty$ title empty$ howpublished empty$
+ month empty$ year empty$ note empty$
+ and and and and and
+ { "all relevant fields are empty in " cite$ * warning$ }
+ 'skip$
+ if$
+ }
+FUNCTION {format.thesis.type}
+ { type duplicate$ empty$
+ 'pop$
+ { swap$ pop$
+ "t" change.case$ "type" bibinfo.check
+ }
+ if$
+}
+FUNCTION {format.tr.number}
+{
+ number "number" bibinfo.check
+ %%type duplicate$ empty$
+ %%{ pop$ bbl.techrep }
+ %%'skip$
+ %%if$
+ %%"type" bibinfo.check
+ %%swap$ duplicate$ empty$
+ %%{ pop$ "t" change.case$ }
+ %%{ tie.or.space.prefix * * }
+ %%if$
+}
+
+FUNCTION {format.org.or.pub}
+{ 't :=
+ ""
+ address empty$ t empty$ and
+ 'skip$
+ {
+ address "address" bibinfo.check *
+ t empty$
+ 'skip$
+ { address empty$
+ 'skip$
+ { ": " * }
+ if$
+ t *
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.publisher.address}
+{ publisher "publisher" bibinfo.warn format.org.or.pub
+}
+
+FUNCTION {format.organization.address}
+{ organization "organization" bibinfo.check format.org.or.pub
+}
+
+FUNCTION {format.institution.address}
+{ institution "institution" bibinfo.check format.org.or.pub
+}
+FUNCTION {format.article.crossref}
+{
+ word.in
+ " \cite{" * crossref * "}" *
+}
+FUNCTION {format.book.crossref}
+{ volume duplicate$ empty$
+ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+ pop$ word.in
+ }
+ { bbl.volume
+ capitalize
+ swap$ tie.or.space.prefix "volume" bibinfo.check * * bbl.of space.word *
+ }
+ if$
+ " \cite{" * crossref * "}" *
+}
+FUNCTION {format.incoll.inproc.crossref}
+{
+ word.in
+ " \cite{" * crossref * "}" *
+}
+FUNCTION {misc}
+{ output.bibitem
+ format.authors "author" output.check
+ format.editors "author and editor" output.check
+ format.title "title" output.check
+ type missing$
+ { skip$ }
+% { format.type "type" output.check }
+ { "type" output.check }
+ %%{ inbrackets type output }
+ if$
+ new.block
+ format.publisher.address output
+ format.date "year" output.check
+ new.block
+ format.note output
+ new.block
+ howpublished new.block.checka
+ howpublished "howpublished" bibinfo.check output
+% output.web.refs % urlbst
+ fin.entry
+ empty.misc.check
+}
+
+FUNCTION {article}
+{ output.bibitem
+ format.authors "author" output.check
+ organization empty$
+ 'skip$
+ { author empty$
+ {
+ format.organizations "organization" output.check
+ }
+ {
+ "; " *
+ no.blank.or.punct
+ format.organizations "organization" output.check
+ }
+ if$
+ }
+ if$
+ new.block
+ format.title "title" output.check
+ type missing$
+ { skip$ }
+ { "type" output.check }
+ if$
+ new.block
+ journal
+ remove.dots
+ ". " *
+ "journal" bibinfo.check
+ "journal" output.check
+ new.block
+ format.journal.date
+ "year" output.check
+ no.blank.or.punct
+ add.semicolon
+ add.blank
+ format.vol.num.pages output
+ new.block
+ format.note output
+% output.web.refs % urlbst
+ fin.entry
+}
+
+FUNCTION {book}
+{ output.bibitem
+ author empty$
+ { editor empty$
+ { format.organizations "organization" output.check }
+ { format.editors "author and editor" output.check }
+ if$
+ }
+ { format.authors output.nonnull
+ "author and editor" editor either.or.check
+ }
+ if$
+ new.block
+ format.btitle "title" output.check
+ format.bvolume output
+ new.block
+ format.edition output
+ new.sentence
+ author empty$ not
+ editor empty$ not
+ and
+ { format.editors "author and editor" output.check }
+ 'skip$
+ if$
+ format.number.series output
+ format.publisher.address output
+ format.date "year" output.check
+ new.block
+ format.note output
+% output.web.refs % urlbst
+ fin.entry
+}
+FUNCTION {booklet}
+{ misc }
+
+FUNCTION {dictionary}
+{ output.bibitem
+ format.booktitle "booktitle" output.check
+ format.bvolume output
+ new.block
+ format.edition output
+ new.sentence
+ format.publisher.address output
+ format.date "year" output.check
+ format.btitle "title" output.check
+ add.semicolon
+ add.blank
+ format.pages "pages" output.check
+ new.block
+ format.note output
+% output.web.refs % urlbst
+ fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ chapter "chapter" output.check
+ new.block
+ format.in.ed.title "title" output.check
+ format.bvolume output
+ format.edition output
+ new.sentence
+ format.number.series output
+ format.publisher.address output
+ format.date "year" output.check
+ date.block
+ add.blank
+ format.pages "pages" output.check
+ new.block
+ format.note output
+% output.web.refs % urlbst
+ fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.edition output
+ new.sentence
+ format.number.series output
+ format.publisher.address output
+ format.date "year" output.check
+ date.block
+% add.blank
+ format.pages "pages" output.check
+ new.block
+ format.note output
+ fin.entry
+}
+FUNCTION {inproceedings}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ new.sentence
+ format.number.series output
+ publisher empty$
+ { format.organization.address output }
+ { organization "organization" bibinfo.check output
+ format.publisher.address output
+ }
+ if$
+ format.date "year" output.check
+ date.block
+ add.blank
+ format.pages "pages" output.check
+ new.block
+ format.note output
+% output.web.refs % urlbst
+ fin.entry
+}
+FUNCTION {conference} { inproceedings }
+FUNCTION {manual}
+{ output.bibitem
+ format.authors output
+ author format.key output
+% add.colon
+ new.block
+ format.btitle "title" output.check
+ organization address new.block.checkb
+ organization "organization" bibinfo.check output
+ address "address" bibinfo.check output
+ format.edition output
+ format.date "year" output.check
+ % new.block ++++ REMOVED (to get comma before note)
+ format.note output
+ fin.entry
+}
+
+FUNCTION {phdthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.btitle "title" output.check
+ new.block
+ "PhD thesis" format.thesis.type output.nonnull
+ school "school" bibinfo.warn output
+% address "address" bibinfo.check output
+ format.date "year" output.check
+% new.block
+% format.note output
+% output.web.refs % urlbst
+ fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+ format.editors output
+ editor format.key output
+% add.colon
+ new.block
+ format.btitle "title" output.check
+ format.bvolume output
+ new.sentence
+ format.number.series output
+ publisher empty$
+ { format.organization.address output }
+ { organization "organization" bibinfo.check output
+ format.publisher.address output
+ }
+ if$
+ format.date "year" output.check
+ % new.block ++++ REMOVED (to get comma before note)
+ format.note output
+ fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.title
+ "title" output.check
+ new.block
+ format.institution.address output
+ format.date "year" output.check
+% format.tr.number output.nonnull
+% new.block
+% format.note output
+% output.web.refs % urlbst
+ fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+ format.authors "author" output.check
+ author format.key output
+% add.colon
+ new.block
+ format.title "title" output.check
+ format.date "year" output.check
+ % new.block ++++ REMOVED (to get comma before note)
+ format.note "note" output.check
+ fin.entry
+}
+
+FUNCTION {default.type} { misc }
+READ
+FUNCTION {sortify}
+{ purify$
+ "l" change.case$
+}
+INTEGERS { len }
+FUNCTION {chop.word}
+{ 's :=
+ 'len :=
+ s #1 len substring$ =
+ { s len #1 + global.max$ substring$ }
+ 's
+ if$
+}
+FUNCTION {format.lab.names}
+{ 's :=
+ "" 't :=
+ s #1 "{vv~}{ll}" format.name$
+ s num.names$ duplicate$
+ #2 >
+ { pop$
+ " " * bbl.etal *
+ }
+ { #2 <
+ 'skip$
+ { s #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" =
+ {
+ " " * bbl.etal *
+ }
+ { bbl.and space.word * s #2 "{vv~}{ll}" format.name$
+ * }
+ if$
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {author.key.label}
+{ author empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.editor.key.label}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.lab.names }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {editor.key.label}
+{ editor empty$
+ { key empty$
+ { cite$ #1 #3 substring$ }
+ 'key
+ if$
+ }
+ { editor format.lab.names }
+ if$
+}
+
+FUNCTION {calc.short.authors}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.label
+ { type$ "proceedings" =
+ 'editor.key.label
+ 'author.key.label
+ if$
+ }
+ if$
+ 'short.list :=
+}
+
+FUNCTION {calc.label}
+{ calc.short.authors
+ short.list
+ "("
+ *
+ year duplicate$ empty$
+ { pop$ "????" }
+ 'skip$
+ if$
+ *
+ 'label :=
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+ #1 'nameptr :=
+ ""
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { s nameptr
+ "{ll{ }}{ ff{ }}{ jj{ }}"
+ format.name$ 't :=
+ nameptr #1 >
+ {
+ " " *
+ namesleft #1 = t "others" = and
+ { "zzzzz" * }
+ { t sortify * }
+ if$
+ }
+ { t sortify * }
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+ "A " #2
+ "An " #3
+ "The " #4 t chop.word
+ chop.word
+ chop.word
+ sortify
+ #1 global.max$ substring$
+}
+FUNCTION {author.sort}
+{ author empty$
+ { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+FUNCTION {author.editor.sort}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { "to sort, need author, editor, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+FUNCTION {editor.sort}
+{ editor empty$
+ { key empty$
+ { "to sort, need editor or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+}
+FUNCTION {presort}
+{ calc.label
+ label sortify
+ " "
+ *
+ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.sort
+ { type$ "proceedings" =
+ 'editor.sort
+ 'author.sort
+ if$
+ }
+ if$
+ #1 entry.max$ substring$
+ 'sort.label :=
+ sort.label
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+
+ITERATE {presort}
+SORT
+STRINGS { last.label next.extra }
+INTEGERS { last.extra.num number.label }
+FUNCTION {initialize.extra.label.stuff}
+{ #0 int.to.chr$ 'last.label :=
+ "" 'next.extra :=
+ #0 'last.extra.num :=
+ #0 'number.label :=
+}
+FUNCTION {forward.pass}
+{ last.label label =
+ { last.extra.num #1 + 'last.extra.num :=
+ last.extra.num int.to.chr$ 'extra.label :=
+ }
+ { "a" chr.to.int$ 'last.extra.num :=
+ "" 'extra.label :=
+ label 'last.label :=
+ }
+ if$
+ number.label #1 + 'number.label :=
+}
+FUNCTION {reverse.pass}
+{ next.extra "b" =
+ { "a" 'extra.label := }
+ 'skip$
+ if$
+ extra.label 'next.extra :=
+ extra.label
+ duplicate$ empty$
+ 'skip$
+ { "{\natexlab{" swap$ * "}}" * }
+ if$
+ 'extra.label :=
+ label extra.label * 'label :=
+}
+EXECUTE {initialize.extra.label.stuff}
+ITERATE {forward.pass}
+REVERSE {reverse.pass}
+FUNCTION {bib.sort.order}
+{ sort.label
+ " "
+ *
+ year field.or.null sortify
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+ITERATE {bib.sort.order}
+SORT
+FUNCTION {begin.bib}
+{ preamble$ empty$
+ 'skip$
+ { preamble$ write$ newline$ }
+ if$
+ "\begin{thebibliography}{" number.label int.to.str$ * "}" *
+ write$ newline$
+ "\providecommand{\natexlab}[1]{#1}"
+ write$ newline$
+% "\providecommand{\url}[1]{\texttt{#1}}"
+% write$ newline$
+% "\providecommand{\urlprefix}{URL }" ++++ EMPTIED by default
+ "\providecommand{\urlprefix}{}"
+ write$ newline$
+ "\providecommand{\doiprefix}{doi: }"
+ write$ newline$
+}
+EXECUTE {begin.bib}
+EXECUTE {init.state.consts}
+ITERATE {call.type$}
+FUNCTION {end.bib}
+{ newline$
+ "\end{thebibliography}" write$ newline$
+}
+EXECUTE {end.bib}
+%% End of customized bst file
+%%
+%% End of file `splncsnat.bst'.