datahandling.tex

% Options for packages loaded elsewhere
\PassOptionsToPackage{unicode}{hyperref}
\PassOptionsToPackage{hyphens}{url}
\PassOptionsToPackage{dvipsnames,svgnames,x11names}{xcolor}
%
\documentclass[
  12pt,
]{style/krantz}
\usepackage{amsmath,amssymb}
\usepackage{lmodern}
\usepackage{iftex}
\ifPDFTeX
  \usepackage[T1]{fontenc}
  \usepackage[utf8]{inputenc}
  \usepackage{textcomp} % provide euro and other symbols
\else % if luatex or xetex
  \usepackage{unicode-math}
  \defaultfontfeatures{Scale=MatchLowercase}
  \defaultfontfeatures[\rmfamily]{Ligatures=TeX,Scale=1}
  \setmonofont[Scale=0.7]{Source Code Pro}
\fi
% Use upquote if available, for straight quotes in verbatim environments
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\IfFileExists{microtype.sty}{% use microtype if available
  \usepackage[]{microtype}
  \UseMicrotypeSet[protrusion]{basicmath} % disable protrusion for tt fonts
}{}
\makeatletter
\@ifundefined{KOMAClassName}{% if non-KOMA class
  \IfFileExists{parskip.sty}{%
    \usepackage{parskip}
  }{% else
    \setlength{\parindent}{0pt}
    \setlength{\parskip}{6pt plus 2pt minus 1pt}}
}{% if KOMA class
  \KOMAoptions{parskip=half}}
\makeatother
\usepackage{xcolor}
\IfFileExists{xurl.sty}{\usepackage{xurl}}{} % add URL line breaks if available
\IfFileExists{bookmark.sty}{\usepackage{bookmark}}{\usepackage{hyperref}}
\hypersetup{
  pdftitle={Data Handling Pocket Reference},
  pdfauthor={Ulrich Matter},
  colorlinks=true,
  linkcolor={Maroon},
  filecolor={Maroon},
  citecolor={Blue},
  urlcolor={Blue},
  pdfcreator={LaTeX via pandoc}}
\urlstyle{same} % disable monospaced font for URLs
\usepackage{color}
\usepackage{fancyvrb}
\newcommand{\VerbBar}{|}
\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
% Add ',fontsize=\small' for more characters per line
\usepackage{framed}
\definecolor{shadecolor}{RGB}{248,248,248}
\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
\newcommand{\AlertTok}[1]{\textcolor[rgb]{0.94,0.16,0.16}{#1}}
\newcommand{\AnnotationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\AttributeTok}[1]{\textcolor[rgb]{0.77,0.63,0.00}{#1}}
\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\BuiltInTok}[1]{#1}
\newcommand{\CharTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
\newcommand{\CommentVarTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\ConstantTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\ControlFlowTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{#1}}
\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\DocumentationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\ErrorTok}[1]{\textcolor[rgb]{0.64,0.00,0.00}{\textbf{#1}}}
\newcommand{\ExtensionTok}[1]{#1}
\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.00,0.00,0.81}{#1}}
\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\ImportTok}[1]{#1}
\newcommand{\InformationTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.13,0.29,0.53}{\textbf{#1}}}
\newcommand{\NormalTok}[1]{#1}
\newcommand{\OperatorTok}[1]{\textcolor[rgb]{0.81,0.36,0.00}{\textbf{#1}}}
\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{#1}}
\newcommand{\PreprocessorTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textit{#1}}}
\newcommand{\RegionMarkerTok}[1]{#1}
\newcommand{\SpecialCharTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\SpecialStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\StringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\VariableTok}[1]{\textcolor[rgb]{0.00,0.00,0.00}{#1}}
\newcommand{\VerbatimStringTok}[1]{\textcolor[rgb]{0.31,0.60,0.02}{#1}}
\newcommand{\WarningTok}[1]{\textcolor[rgb]{0.56,0.35,0.01}{\textbf{\textit{#1}}}}
\usepackage{longtable,booktabs,array}
\usepackage{calc} % for calculating minipage widths
% Correct order of tables after \paragraph or \subparagraph
\usepackage{etoolbox}
\makeatletter
\patchcmd\longtable{\par}{\if@noskipsec\mbox{}\fi\par}{}{}
\makeatother
% Allow footnotes in longtable head/foot
\IfFileExists{footnotehyper.sty}{\usepackage{footnotehyper}}{\usepackage{footnote}}
\makesavenoteenv{longtable}
\usepackage{graphicx}
\makeatletter
\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
\makeatother
% Scale images if necessary, so that they will not overflow the page
% margins by default, and it is still possible to overwrite the defaults
% using explicit options in \includegraphics[width, height, ...]{}
\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
% Set default figure placement to htbp
\makeatletter
\def\fps@figure{htbp}
\makeatother
\setlength{\emergencystretch}{3em} % prevent overfull lines
\providecommand{\tightlist}{%
  \setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
\setcounter{secnumdepth}{5}

\usepackage{float}
\usepackage{booktabs}
\usepackage{longtable}
\usepackage[bf,singlelinecheck=off]{caption}

\usepackage{Alegreya}
\usepackage[scale=.7]{sourcecodepro}

\usepackage{framed,color}
\definecolor{shadecolor}{RGB}{248,248,248}

\renewcommand{\textfraction}{0.05}
\renewcommand{\topfraction}{0.8}
\renewcommand{\bottomfraction}{0.8}
\renewcommand{\floatpagefraction}{0.75}

\renewenvironment{quote}{\begin{VF}}{\end{VF}}
\let\oldhref\href
\renewcommand{\href}[2]{#2\footnote{\url{#1}}}

\ifxetex
  \usepackage{letltxmacro}
  \setlength{\XeTeXLinkMargin}{1pt}
  \LetLtxMacro\SavedIncludeGraphics\includegraphics
  \def\includegraphics#1#{% #1 catches optional stuff (star/opt. arg.)
    \IncludeGraphicsAux{#1}%
  }%
  \newcommand*{\IncludeGraphicsAux}[2]{%
    \XeTeXLinkBox{%
      \SavedIncludeGraphics#1{#2}%
    }%
  }%
\fi

\makeatletter
\newenvironment{kframe}{%
\medskip{}
\setlength{\fboxsep}{.8em}
 \def\at@end@of@kframe{}%
 \ifinner\ifhmode%
  \def\at@end@of@kframe{\end{minipage}}%
  \begin{minipage}{\columnwidth}%
 \fi\fi%
 \def\FrameCommand##1{\hskip\@totalleftmargin \hskip-\fboxsep
 \colorbox{shadecolor}{##1}\hskip-\fboxsep
     % There is no \\@totalrightmargin, so:
     \hskip-\linewidth \hskip-\@totalleftmargin \hskip\columnwidth}%
 \MakeFramed {\advance\hsize-\width
   \@totalleftmargin\z@ \linewidth\hsize
   \@setminipage}}%
 {\par\unskip\endMakeFramed%
 \at@end@of@kframe}
\makeatother

\makeatletter
\@ifundefined{Shaded}{
}{\renewenvironment{Shaded}{\begin{kframe}}{\end{kframe}}}
\makeatother

\newenvironment{rmdblock}[1]
  {
  \begin{itemize}
  \renewcommand{\labelitemi}{
    \raisebox{-.7\height}[0pt][0pt]{
      {\setkeys{Gin}{width=3em,keepaspectratio}\includegraphics{images/#1}}
    }
  }
  \setlength{\fboxsep}{1em}
  \begin{kframe}
  \item
  }
  {
  \end{kframe}
  \end{itemize}
  }
\newenvironment{rmdnote}
  {\begin{rmdblock}{note}}
  {\end{rmdblock}}
\newenvironment{rmdcaution}
  {\begin{rmdblock}{caution}}
  {\end{rmdblock}}
\newenvironment{rmdimportant}
  {\begin{rmdblock}{important}}
  {\end{rmdblock}}
\newenvironment{rmdtip}
  {\begin{rmdblock}{tip}}
  {\end{rmdblock}}
\newenvironment{rmdwarning}
  {\begin{rmdblock}{warning}}
  {\end{rmdblock}}

\usepackage{makeidx}
\makeindex

\urlstyle{tt}

\usepackage{amsthm}
\makeatletter
\def\thm@space@setup{%
  \thm@preskip=8pt plus 2pt minus 4pt
  \thm@postskip=\thm@preskip
}
\makeatother

\frontmatter
\ifLuaTeX
  \usepackage{selnolig}  % disable illegal ligatures
\fi
\usepackage[]{natbib}
\bibliographystyle{apalike}

\title{Data Handling Pocket Reference}
\author{Ulrich Matter}
\date{2022-10-18}

\begin{document}
\maketitle

% leave a few empty pages before the dedication page
%\cleardoublepage\newpage\thispagestyle{empty}\null
%\cleardoublepage\newpage\thispagestyle{empty}\null
%\cleardoublepage\newpage


\thispagestyle{empty}

\begin{center}
\includegraphics{img/dedication.pdf}
\end{center}

\setlength{\abovedisplayskip}{-5pt}
\setlength{\abovedisplayshortskip}{-5pt}

{
\hypersetup{linkcolor=}
\setcounter{tocdepth}{2}
\tableofcontents
}
\listoffigures
\listoftables
\hypertarget{preface}{%
\chapter*{Preface}\label{preface}}


In applied econometric research and business analytics alike, many steps in handling digital data are involved before running regression estimations and training machine learning algorithms. While often neglected in statistics and econometrics curricula, the steps of gathering, cleaning, storing, and filtering data for research/analytics purposes are of utmost importance to ensure accurate and reproducible analytic insights. This pocket reference first introduces and summarizes foundational and practically relevant data and data processing concepts, and then guides the reader through each step of how to get from the raw data to the final data analysis output.

\hypertarget{prerequisites-and-aims}{%
\section*{Prerequisites and aims}\label{prerequisites-and-aims}}


The book presupposes a basic knowledge of undergraduate economics and statistics and relates several case studies to practical questions in these fields. Finally, the aim is to give you firsthand practical insights into each part of the data science pipeline in the context of business and economics research.

\begin{figure}
\centering
\includegraphics{img/cc.png}
\caption{Creative Commons License}
\end{figure}

The online version of this book is licensed under the \href{http://creativecommons.org/licenses/by-nc-sa/4.0/}{Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License}.

\begin{flushright}
Ulrich Matter
St.~Gallen, Switzerland
\end{flushright}

\mainmatter

\hypertarget{introduction}{%
\chapter{Introduction}\label{introduction}}

Lower computing costs, a stark decrease in storage costs for digital data, as well as the diffusion of the Internet, have led to the development of new products (e.g., smartphones) and services (e.g., web search engines, cloud computing) over the last few decades. A side product of these developments is a strong increase in the availability of digital data describing all kinds of everyday human activities \citep{einav_levin2014, matter_stutzer2015}. As a consequence, new business models and economic structures are emerging with data as their core commodity (i.e., AI-related technological and economic change). For example, the current hype surrounding `Artificial Intelligence' (AI) - largely fueled by the broad application of machine-learning techniques such as `deep learning' (a form of artificial neural networks) - would not be conceivable without the increasing abundance of large amounts of digital data on all kind of socio-economic entities and activities. In short, without understanding and handling the underlying data streams properly, the AI-driven economy cannot function. The same rationale applies, of course, to other ways of making use of digital data. Be it traditional big data analytics or scientific research (e.g., applied econometrics).

The need for proper handling of large amounts of digital data has given rise to the interdisciplinary field of \href{https://en.wikipedia.org/wiki/Data_science}{`Data Science'} and increasing demand for `Data Scientists'. While nothing within Data Science is particularly new on its own, the combination of skills and insights from different fields (particularly Computer Science and Statistics) has proven to be very productive in meeting new challenges posed by a data-driven economy. In that sense, Data Science is rather a craft than a scientific field. As such, it presupposes a more practical and broader understanding of the data than traditional Computer Science and Statistics from which Data Science borrows its methods. This book focuses on the skills necessary for \emph{acquiring, cleaning, and manipulating} digital data for research/analytics purposes.

\hypertarget{programming-with-data}{%
\chapter{Programming with Data}\label{programming-with-data}}

\hypertarget{handling-data-programmatically}{%
\section{Handling data programmatically}\label{handling-data-programmatically}}

The need for proper handling of large amounts of digital data has given rise to the interdisciplinary field of \href{https://en.wikipedia.org/wiki/Data_science}{`Data Science'} as well as an increasing demand for `Data Scientists'. While nothing within Data Science is particularly new on its own, it is the combination of skills and insights from different fields (particularly Computer Science and Statistics) that has proven to be very productive in meeting new challenges posed by a data-driven economy. The various facets of this new craft are often illustrated in the `Data Science' Venn diagram, reflecting the combination of knowledge and skills from Mathematics/Statistics, substantive expertise in the particular scientific field in which Data Science is applied (here: Economics), and the skills for \emph{acquiring, cleaning, and analyzing} data \emph{programmatically}.

\begin{figure}

{\centering \includegraphics[width=0.6\linewidth]{img/venn_diagramm} 

}

\caption{Venn diagram illustrating the domains of Data Science in the context of Economics.}\label{fig:venn}
\end{figure}



In the proposed framework of Data Science in Economics followed in this book, programming skills, basic knowledge about computing and in data technologies serve as a complement to engaging in modern economic analysis. They are necessary ingredients to both working new econometric approaches in machine learning (and the preceding feature engineering) as well as to solving complex problems in the domain of economic modeling. Moreover, programming (or `coding') is the basis to better understand and engage with the handling of data for analytics purposes.

While industry-scale data science projects tend to include data processing frameworks and programming languages (such as Scala, Python, and SQL), we will see that the core steps of how to get from the raw data to the final analysis report can for most simpler data projects be easily managed with just one programming language. In the case of this book, we choose \emph{R}.

\hypertarget{why-r}{%
\section{\texorpdfstring{Why \emph{R}?}{Why R?}}\label{why-r}}

\hypertarget{the-data-language}{%
\subsection{The `data language'}\label{the-data-language}}

The programming language and open-source statistical computing environment \href{www.r-project.org}{\emph{R}} has over the last decade become a core tool for data science in industry and academia. It was originally designed as a tool for statistical analysis. Many characteristics of the language make \emph{R} particularly useful to work with data. \emph{R} is increasingly used in various domains, going well beyond the traditional applications of academic research.

\hypertarget{high-level-language-relatively-easy-to-learn}{%
\subsection{High-level language, relatively easy to learn}\label{high-level-language-relatively-easy-to-learn}}

\emph{R} is a relatively easy computer language to learn for people with no previous programming experience. The syntax is rather intuitive and error messages are not too cryptic to understand (this facilitates learning by doing). Moreover, with \emph{R}'s recent stark rise in popularity, there are plenty of freely accessible resources online that help beginners to learn the language.

\hypertarget{free-open-source-large-community}{%
\subsection{Free, open source, large community}\label{free-open-source-large-community}}

Due to its vast base of contributors, \emph{R} serves as a valuable tool for users in various fields related to data analysis and computation (economics/econometrics, biomedicine, business analytics, etc.). \emph{R} users have direct access to thousands of freely available `\emph{R}-packages' (small software libraries written in \emph{R}), covering diverse aspects of data analysis, statistics, data preparation, and data import.

Hence, a lot of people using \emph{R} as a tool in their daily work do not actually `write programs' (in the traditional sense of the word), but apply \emph{R} packages. Applied econometrics with \emph{R} is a good example of this. Almost any function a modern commercial computing environment with a focus on statistics and econometrics (such as \href{http://www.stata.com/}{STATA}) is offering, can also be found within the \emph{R} environment. Furthermore, there are \emph{R} packages covering all the areas of modern data analytics, including natural language processing, machine learning, big data analytics, etc. (see the \href{https://cran.r-project.org/web/views/}{CRAN Task Views} for an overview). We thus do not actually have to write a program for many tasks we perform with \emph{R}. Instead, we can build on already existing and reliable packages.

\hypertarget{rrstudio-overview}{%
\section{\texorpdfstring{\emph{R}/RStudio overview}{R/RStudio overview}}\label{rrstudio-overview}}

\emph{R} is the high-level (meaning `more user friendly') programming language for statistical computing. Once we have installed \emph{R} on our computer, we can run it\ldots{}

\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\tightlist
\item
  \ldots directly from the command line, by typing \texttt{R} and hit enter (here in the OSX terminal):
\end{enumerate}

\begin{figure}

{\centering \includegraphics[width=0.6\linewidth]{img/r_terminal} 

}

\caption{Running \emph{R} in the Mac/OSX terminal.}\label{fig:terminal}
\end{figure}



\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\setcounter{enumi}{1}
\tightlist
\item
  \ldots with the simple \href{https://en.wikipedia.org/wiki/Integrated_development_environment}{Integrated Development Environment (IDE)} delivered with the basic \emph{R} installation
\end{enumerate}

\begin{figure}

{\centering \includegraphics[width=0.75\linewidth]{img/r_ide} 

}

\caption{Running \emph{R} in the original \emph{R} GUI/IDE.}\label{fig:ide}
\end{figure}



\begin{enumerate}
\def\labelenumi{\alph{enumi}.}
\setcounter{enumi}{2}
\tightlist
\item
  \ldots or with the more elaborated and user-friendly IDE called \emph{RStudio} (either locally or in the cloud, see, for example \href{https://rstudio.cloud/}{RStudio Cloud}:
\end{enumerate}

\begin{figure}

{\centering \includegraphics[width=0.75\linewidth]{img/rstudio_panels} 

}

\caption{Running \emph{R} in RStudio (IDE).}\label{fig:rstudio}
\end{figure}



The latter is what we will do throughout this course. RStudio is a very helpful tool for simple data analysis with \emph{R}, writing \emph{R} scripts (short \emph{R} programs), or even for developing \emph{R} packages (software written in \emph{R}), as well as building interactive documents, presentations, etc. Moreover, it offers many options to change its own appearance (Pane Layout, Code Highlighting, etc.).

In the following, we have a look at each of the main panels that will be relevant in this course.

\hypertarget{the-r-console}{%
\subsection{\texorpdfstring{The \emph{R}-Console}{The R-Console}}\label{the-r-console}}

When working in an interactive session, we simply type \emph{R} commands directly into the \emph{R} console. Typically, the output of executing a command this way is also directly printed to the console. Hence, we type a command on one line, hit enter, and the output is presented on the next line.

\begin{figure}

{\centering \includegraphics[width=0.75\linewidth]{img/rstudio_console} 

}

\caption{Running \emph{R} in the Mac/OSX terminal.}\label{fig:console}
\end{figure}



For example, we can tell \emph{R} to print the phrase \texttt{Hello\ world} to the console, by typing to following command in the console and hit enter:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{print}\NormalTok{(}\StringTok{"Hello world"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "Hello world"
\end{verbatim}

\hypertarget{r-scripts}{%
\subsection{\texorpdfstring{\emph{R}-Scripts}{R-Scripts}}\label{r-scripts}}

Apart from very short interactive sessions, it usually makes sense to write \emph{R} code not directly in the command line but to an \emph{R}-script in the script panel. This way, we can easily execute several lines at once, comment the code (to explain what it does), save it on our hard disk, and further develop the code later on.

\begin{figure}

{\centering \includegraphics[width=0.75\linewidth]{img/rstudio_script} 

}

\caption{The \emph{R} Script window in RStudio.}\label{fig:rscript}
\end{figure}



\hypertarget{r-environment}{%
\subsection{\texorpdfstring{\emph{R} Environment}{R Environment}}\label{r-environment}}

The environment pane shows what variables, objects, and data are loaded in our current \emph{R} session. Moreover, it offers functions to open documents and import data.

\begin{figure}

{\centering \includegraphics[width=0.5\linewidth]{img/rstudio_environment} 

}

\caption{The environment window in RStudio.}\label{fig:renvironment}
\end{figure}



\hypertarget{file-browser}{%
\subsection{File Browser}\label{file-browser}}

With the file browser window we can navigate through the folder structure and files on our computer's hard disk, modify files, and set the working directory of our current \emph{R} session. Moreover, it has a pane to show plots generated in \emph{R} and a pane with help pages and \emph{R} documentation.

\begin{figure}

{\centering \includegraphics[width=0.5\linewidth]{img/rstudio_files} 

}

\caption{The file browser window in RStudio.}\label{fig:rfiles}
\end{figure}



\hypertarget{first-steps-with-r}{%
\section{First steps with R}\label{first-steps-with-r}}

Before introducing some of the key functions and packages for data handling and data analysis with R, we should understand how such programs basically work and how we can write them in R. Once we understand the basics of the R language and how to write simple programs, understanding and applying already implemented programs is much easier.\footnote{In fact, since R is an open source environment, you can directly look at already implemented programs in order to learn how they work.}

\hypertarget{values-vectors-and-variables}{%
\subsection{Values, Vectors, and Variables}\label{values-vectors-and-variables}}

The simplest objects to work with in R are vectors. In fact, even a simple numeric value such as \texttt{5.5} or a string of characters (text) like \texttt{"Hello"} is considered a vector (a scalar).\footnote{You can try this out in the R console by typing \texttt{is.vector(5.5)} and \texttt{is.vector("Hello")}.}

A first good step to get familiar with coding in R is to assign names to the objects/values you are working with. For example, when summing up two numeric values, you might want to store the result in a separate object and call this object \texttt{result}. This is done with the assignment operator \texttt{\textless{}-} (or \texttt{=}, which serves the same purpose).

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# assign the variable name "result" to the sum of two numeric values}
\NormalTok{result }\OtherTok{\textless{}{-}} \FloatTok{25.6} \SpecialCharTok{+} \FloatTok{53.4} 
\end{Highlighting}
\end{Shaded}

Whenever you want to re-use the just computed sum, you can directly call the object by its name (the variable \texttt{result}):

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# check what "is stored in" result}
\NormalTok{result}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 79
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# further work with the value in result}
\NormalTok{result }\SpecialCharTok{{-}} \DecValTok{20}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 59
\end{verbatim}

With the combine-function (\texttt{c()}), you easily form vectors with several elements and name the elements in the vector. By doing so, you create a very simple dataset. For example, suppose you survey the age of a sample of persons. The age values (in years) gives you an integer vector. By naming each integer value (each vector element) after the corresponding person's name, you then have a simple dataset stored in a named R vector.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# a simple integer vector}
\NormalTok{a }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\DecValTok{10}\NormalTok{,}\DecValTok{22}\NormalTok{,}\DecValTok{33}\NormalTok{, }\DecValTok{22}\NormalTok{, }\DecValTok{40}\NormalTok{)}

\CommentTok{\# give names to vector elements}
\FunctionTok{names}\NormalTok{(a) }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\StringTok{"Andy"}\NormalTok{, }\StringTok{"Betty"}\NormalTok{, }\StringTok{"Claire"}\NormalTok{, }\StringTok{"Daniel"}\NormalTok{, }\StringTok{"Eva"}\NormalTok{)}
\NormalTok{a}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   Andy  Betty Claire Daniel    Eva 
##     10     22     33     22     40
\end{verbatim}

To retrieve specific values from this vector, you can either select the corresponding vector element with the element's index (the first, second, third, etc. element) or via its name.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# indexing either via a number of vector element (start count with 1)}
\CommentTok{\# or by element name}
\NormalTok{a[}\DecValTok{3}\NormalTok{]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Claire 
##     33
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{a[}\StringTok{"Claire"}\NormalTok{]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Claire 
##     33
\end{verbatim}

When not sure what kind of object \texttt{a} is, the \texttt{str()} (structure) function, provides you with a short summary.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# inspect the object you are working with}
\FunctionTok{str}\NormalTok{(a) }\CommentTok{\# returns the structure of the object ("what is in variable a?")}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##  Named num [1:5] 10 22 33 22 40
##  - attr(*, "names")= chr [1:5] "Andy" "Betty" "Claire" "Daniel" ...
\end{verbatim}

If you want to learn more about what the \texttt{c()} or \texttt{str()} functions (or any other pre-defined R functions) do and how they should be used, type \texttt{help(FUNCTION-NAME)} or \texttt{?FUNCTION-NAME} in the console and hit enter. A help-page with detailed explanations of what the function is for and how it can be used will appear in one of the R-Studio panels.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{help}\NormalTok{(str)}
\NormalTok{?c}
\end{Highlighting}
\end{Shaded}

\hypertarget{math-operators}{%
\subsection{Math operators}\label{math-operators}}

Above, we have just in a side remark introduced the very intuitive syntax for two common math operators in R: \texttt{+} for the addition of numeric or integer values, and \texttt{-} for subtraction. R knows all basic math operators and has a variety of functions to handle more advanced mathematical problems. One basic practical application of R in academic life is to use it as a sophisticated (and programmable) calculator.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# basic arithmetic}
\DecValTok{2}\SpecialCharTok{+}\DecValTok{2}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 4
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{sum\_result }\OtherTok{\textless{}{-}} \DecValTok{2}\SpecialCharTok{+}\DecValTok{2}
\NormalTok{sum\_result}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 4
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{sum\_result }\SpecialCharTok{{-}}\DecValTok{2}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 2
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DecValTok{4}\SpecialCharTok{*}\DecValTok{5}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 20
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DecValTok{20}\SpecialCharTok{/}\DecValTok{5}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 4
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# order of operations}
\DecValTok{2}\SpecialCharTok{+}\DecValTok{2}\SpecialCharTok{*}\DecValTok{3}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 8
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{(}\DecValTok{2}\SpecialCharTok{+}\DecValTok{2}\NormalTok{)}\SpecialCharTok{*}\DecValTok{3}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 12
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{(}\DecValTok{5}\SpecialCharTok{+}\DecValTok{5}\NormalTok{)}\SpecialCharTok{/}\NormalTok{(}\DecValTok{2}\SpecialCharTok{+}\DecValTok{3}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 2
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# work with variables}
\NormalTok{a }\OtherTok{\textless{}{-}} \DecValTok{20}
\NormalTok{b }\OtherTok{\textless{}{-}} \DecValTok{10}
\NormalTok{a}\SpecialCharTok{/}\NormalTok{b}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 2
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# arithmetics with vectors}
\NormalTok{a }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\DecValTok{1}\NormalTok{,}\DecValTok{4}\NormalTok{,}\DecValTok{6}\NormalTok{)}
\NormalTok{a }\SpecialCharTok{*} \DecValTok{2}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1]  2  8 12
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{b }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\DecValTok{10}\NormalTok{,}\DecValTok{40}\NormalTok{,}\DecValTok{80}\NormalTok{)}
\NormalTok{a }\SpecialCharTok{*}\NormalTok{ b}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1]  10 160 480
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{a }\SpecialCharTok{+}\NormalTok{ b}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 11 44 86
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# other common math operators and functions}
\DecValTok{4}\SpecialCharTok{\^{}}\DecValTok{2}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 16
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{sqrt}\NormalTok{(}\DecValTok{4}\SpecialCharTok{\^{}}\DecValTok{2}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 4
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{log}\NormalTok{(}\DecValTok{2}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 0.6931
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{exp}\NormalTok{(}\DecValTok{10}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 22026
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{log}\NormalTok{(}\FunctionTok{exp}\NormalTok{(}\DecValTok{10}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 10
\end{verbatim}

To look up the most common math operators in R and get more details about how to use them type

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{?}\StringTok{\textasciigrave{}}\AttributeTok{+}\StringTok{\textasciigrave{}}
\end{Highlighting}
\end{Shaded}

in the R console and hit enter.

\hypertarget{basic-programming-concepts-in-r}{%
\section{Basic programming concepts in R}\label{basic-programming-concepts-in-r}}

In very simple terms, programming/coding is all about using a computer language to instruct a computer what to do. What reads very complex at first sight, is actually rather simple in (at least for a large array of basic programming problems). At the core of almost any R program is the right application and combination of just a handful of basic programming concepts: loops, logical statements, control statements, and functions. Once you a) conceptually understand what these concepts are for, and b) have learned the syntax of how to use these concepts when writing a program in R, addressing all kind of data handling problems efficiently with R, will simply become a matter of training/practice.

\hypertarget{loops}{%
\subsection{Loops}\label{loops}}

A loop is typically a sequence of statements executed a specific number of times. How often the code `inside' the loop is executed depends on a clearly defined control statement. If we know in advance how often the code inside the loop has to be executed, we typically write a so-called `for-loop'. We typically write a so-called' while-loop' if the number of iterations is not clearly known before executing the code. The following subsections illustrate both of these concepts in R.

\hypertarget{for-loops}{%
\subsection{For-loops}\label{for-loops}}

In simple terms, a for-loop tells the computer to execute a sequence of commands `for each case in a set of n cases'. The flowchart in Figure @ref(fig: for) illustrates the concept.

\begin{figure}

{\centering \includegraphics[width=0.4\linewidth]{img/forloop} 

}

\caption{For-loop illustration.}\label{fig:for}
\end{figure}



For example, a for-loop could be used, to sum up each element in a numeric vector of fixed length (thus, the number of iterations is clearly defined). In plain English, the for-loop would state something like: ``Start with 0 as the current total value, for each of the elements in the vector, add the value of this element to the current total value.'' Note how this logically implies that the loop will `stop' once the value of the last element in the vector is added to the total. Let's illustrate this in R. Take the numeric vector \texttt{c(1,2,3,4,5)}. A for loop to sum up all elements can be implemented as follows:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# vector to be summed up}
\NormalTok{numbers }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\DecValTok{1}\NormalTok{,}\FloatTok{2.1}\NormalTok{,}\FloatTok{3.5}\NormalTok{,}\FloatTok{4.8}\NormalTok{,}\DecValTok{5}\NormalTok{)}
\CommentTok{\# initiate total}
\NormalTok{total\_sum }\OtherTok{\textless{}{-}} \DecValTok{0}
\CommentTok{\# number of iterations}
\NormalTok{n }\OtherTok{\textless{}{-}} \FunctionTok{length}\NormalTok{(numbers)}
\CommentTok{\# start loop}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in} \DecValTok{1}\SpecialCharTok{:}\NormalTok{n) \{}
\NormalTok{     total\_sum }\OtherTok{\textless{}{-}}\NormalTok{ total\_sum }\SpecialCharTok{+}\NormalTok{ numbers[i]}
\NormalTok{\}}

\CommentTok{\# check result}
\NormalTok{total\_sum}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 16.4
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# compare with the result of sum() function}
\FunctionTok{sum}\NormalTok{(numbers)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 16.4
\end{verbatim}

\hypertarget{nested-for-loops}{%
\subsubsection{Nested for-loops}\label{nested-for-loops}}

In some situations, a simple for-loop might not be sufficient. Within one sequence of commands, there might be another sequence of commands that also has to be executed for a number of times each time the first sequence of commands is executed. In such a case, we speak of a `nested for-loop'. We can illustrate this easily by extending the example of the numeric vector above to a matrix for which we want to sum up the values in each column. Building on the loop implemented above, we would say `for each column \texttt{j} of a given numeric matrix, execute the for-loop defined above'.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# matrix to be summed up}
\NormalTok{numbers\_matrix }\OtherTok{\textless{}{-}} \FunctionTok{matrix}\NormalTok{(}\DecValTok{1}\SpecialCharTok{:}\DecValTok{20}\NormalTok{, }\AttributeTok{ncol =} \DecValTok{4}\NormalTok{)}
\NormalTok{numbers\_matrix}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##      [,1] [,2] [,3] [,4]
## [1,]    1    6   11   16
## [2,]    2    7   12   17
## [3,]    3    8   13   18
## [4,]    4    9   14   19
## [5,]    5   10   15   20
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# number of iterations for the outer loop}
\NormalTok{m }\OtherTok{\textless{}{-}} \FunctionTok{ncol}\NormalTok{(numbers\_matrix)}
\CommentTok{\# number of iterations for the inner loop}
\NormalTok{n }\OtherTok{\textless{}{-}} \FunctionTok{nrow}\NormalTok{(numbers\_matrix)}
\CommentTok{\# start outer loop (loop over columns of the matrix)}
\ControlFlowTok{for}\NormalTok{ (j }\ControlFlowTok{in} \DecValTok{1}\SpecialCharTok{:}\NormalTok{m) \{}
     \CommentTok{\# start inner loop}
     \CommentTok{\# initiate total}
\NormalTok{     total\_sum }\OtherTok{\textless{}{-}} \DecValTok{0}
     \ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in} \DecValTok{1}\SpecialCharTok{:}\NormalTok{n) \{}
\NormalTok{          total\_sum }\OtherTok{\textless{}{-}}\NormalTok{ total\_sum }\SpecialCharTok{+}\NormalTok{ numbers\_matrix[i, j]}
\NormalTok{          \}}
     \FunctionTok{print}\NormalTok{(total\_sum)}
\NormalTok{     \}}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 15
## [1] 40
## [1] 65
## [1] 90
\end{verbatim}

\hypertarget{while-loop}{%
\subsubsection{While-loop}\label{while-loop}}

In a situation where a program has to repeatedly run a sequence of commands, but we don't know in advance how many iterations we need to reach the intended goal, a while-loop can help. In simple terms, a while loop keeps executing a sequence of commands as long as a certain logical statement is true. The flow chart in Figure @ref(fig: while) illustrates this point.

\begin{figure}

{\centering \includegraphics[width=0.7\linewidth]{img/while_loop_own} 

}

\caption{While-loop illustration.}\label{fig:while}
\end{figure}



For example, a while-loop in plain English could state something like ``start with 0 as the total, add 1.12 to the total until the total is larger than 20.'' We can implement this in R as follows.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# initiate starting value}
\NormalTok{total }\OtherTok{\textless{}{-}} \DecValTok{0}
\CommentTok{\# start loop}
\ControlFlowTok{while}\NormalTok{ (total }\SpecialCharTok{\textless{}=} \DecValTok{20}\NormalTok{) \{}
\NormalTok{     total }\OtherTok{\textless{}{-}}\NormalTok{ total }\SpecialCharTok{+} \FloatTok{1.12}
\NormalTok{\}}

\CommentTok{\# check the result}
\NormalTok{total}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 20.16
\end{verbatim}

\hypertarget{booleans-and-logical-statements}{%
\subsection{Booleans and logical statements}\label{booleans-and-logical-statements}}

Note that in order to write a meaningful while-loop we have to make use of a logical statement such as ``the value stored in the variable \texttt{total}is smaller or equal to \texttt{20}'' (\texttt{total\ \textless{}=\ 20}). A logical statement results in a `Boolean' data type. That is, a data type with the only two possible values \texttt{TRUE} or \texttt{FALSE} (\texttt{1} or \texttt{0}).

\begin{Shaded}
\begin{Highlighting}[]
\DecValTok{2}\SpecialCharTok{+}\DecValTok{2} \SpecialCharTok{==} \DecValTok{4}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] TRUE
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\DecValTok{3}\SpecialCharTok{+}\DecValTok{3} \SpecialCharTok{==} \DecValTok{7}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] FALSE
\end{verbatim}

Logical statements play an important role in fundamental programming concepts. In particular, they are crucial to make conditional statements (`if-statements') that build the control structure of a program, controlling the `direction' the program takes (given certain conditions).

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{condition }\OtherTok{\textless{}{-}} \ConstantTok{TRUE}
\ControlFlowTok{if}\NormalTok{ (condition) \{}
     \FunctionTok{print}\NormalTok{(}\StringTok{"This is true!"}\NormalTok{)}
\NormalTok{\} }\ControlFlowTok{else}\NormalTok{ \{}
     \FunctionTok{print}\NormalTok{(}\StringTok{"This is false!"}\NormalTok{)}
\NormalTok{\}}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "This is true!"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{condition }\OtherTok{\textless{}{-}} \ConstantTok{FALSE}
\ControlFlowTok{if}\NormalTok{ (condition) \{}
     \FunctionTok{print}\NormalTok{(}\StringTok{"This is true!"}\NormalTok{)}
\NormalTok{\} }\ControlFlowTok{else}\NormalTok{ \{}
     \FunctionTok{print}\NormalTok{(}\StringTok{"This is false!"}\NormalTok{)}
\NormalTok{\}}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "This is false!"
\end{verbatim}

\hypertarget{r-functions}{%
\subsection{R functions}\label{r-functions}}

R programs heavily rely on functions. Conceptually, `functions' in R are very similar to what we know as `functions' in math (i.e., \(f:X \rightarrow Y\)). A function can thus, e.g., take a variable \(X\) as input and provide value \(Y\) as output. The actual calculation of \(Y\) based on \(X\) can be something as simple as \(2\times X = Y\). But it could also be a very complex algorithm or an operation that does not directly have anything to do with numbers and arithmetic.\footnote{Of course, on the very low level, everything that happens in a microprocessor can, in the end, be expressed in some formal way using math. However, the point here is that at the level we work with R, a function could simply process different text strings (i.e., stack them together). Thus for us as programmers, R functions do not necessarily have to do anything with arithmetic and numbers but could serve all kinds of purposes, including the parsing of HTML code, etc.}

In R---and many other programming languages---functions take `parameter values' as input, process those values according to a predefined program, and `return' the result. For example, a function could take a numeric vector as input and return the sum of all the individual numeric values in the input vector.

When we open RStudio, all basic functions are already loaded automatically. This means we can directly call them from the R-Console or by executing an R-Script. As R is made for data analysis and statistics, the basic functions loaded with R cover many aspects of tasks related to working with and analyzing data. Besides these basic functions, thousands of additional functions covering all kinds of topics related to data analysis can be loaded additionally by installing the respective R-packages (\texttt{install.\ packages("PACKAGE-NAME")}) and then loading the packages with \texttt{library(PACKAGE-NAME)}. In addition, it is straightforward to define our own functions.

\hypertarget{case-study-compute-the-mean}{%
\subsubsection{Case study: Compute the mean}\label{case-study-compute-the-mean}}

To illustrate the point of how functions work in R and how we can write our own functions in R, the following code-example illustrates how to implement a function that computes the mean/average value, given a numeric vector.

First, we initiate a simple numeric vector which we then use as an example to test the function. Whenever you implement a function, it is very useful to first define a simple example of an input for which you know what the output should be.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# a simple integer vector, for which we want to compute the Mean}
\NormalTok{a }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\FloatTok{5.5}\NormalTok{, }\FloatTok{7.5}\NormalTok{)}
\CommentTok{\# desired functionality and output:}
\CommentTok{\# my\_mean(a)}
\CommentTok{\# 6.5}
\end{Highlighting}
\end{Shaded}

In this example, we would thus expect the output to be \texttt{6.5}. Later, we will compare the output of our function with this in order to check whether our function works as desired.

In addition to defining a simple example and the desired output, it makes sense to also think about \emph{how} the function is expected to produce this output. When implementing functions related to statistics (such as the mean), it usually makes sense to have a look at the mathematical definition:

\(\bar{x} = \frac{1}{n}\left (\sum_{i=1}^n{x_i}\right ) = \frac{x_1+x_2+\cdots +x_n}{n}\).

Now, we can start thinking about implementing the function based on built-in R functions. From looking at the mathematical definition of the mean (\(\bar{x}\)), we recognize that there are two main components to computing the mean:

\begin{itemize}
\tightlist
\item
  \(\sum_{i=1}^n{x_i}\): the sum of all the elements in vector \(x\)
\item
  \(n\): the number of elements in vector \(x\).
\end{itemize}

Once we know how to get these two components, computing the mean is straightforward. In R, there are two built-in functions that deliver exactly these two components:

\begin{itemize}
\tightlist
\item
  \texttt{sum()} returns the sum of all the values in I ts arguments (i.e., if \texttt{x} is a numeric vector, \texttt{sum(x)} returns the sum of all elements in \texttt{x}).
\item
  \texttt{length()} returns the length of a given vector.
\end{itemize}

With the following short line of code, we thus get the mean of the elements in vector \texttt{a}.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{sum}\NormalTok{(a)}\SpecialCharTok{/}\FunctionTok{length}\NormalTok{(a)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 6.5
\end{verbatim}

All that is left to do is to pack all this into the function body of our newly defined \texttt{my\_mean()} function:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# define our own function to compute the mean, given a numeric vector}
\NormalTok{my\_mean }\OtherTok{\textless{}{-}} \ControlFlowTok{function}\NormalTok{(x) \{}
\NormalTok{     x\_bar }\OtherTok{\textless{}{-}} \FunctionTok{sum}\NormalTok{(x) }\SpecialCharTok{/} \FunctionTok{length}\NormalTok{(x)}
     \FunctionTok{return}\NormalTok{(x\_bar)}
\NormalTok{\}}
\end{Highlighting}
\end{Shaded}

Now we can test it based on our example:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# test it}
\FunctionTok{my\_mean}\NormalTok{(a)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 6.5
\end{verbatim}

Moreover, we can test it by comparing it with the built-in \texttt{mean()} function:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{b }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\DecValTok{4}\NormalTok{,}\DecValTok{5}\NormalTok{,}\DecValTok{2}\NormalTok{,}\DecValTok{5}\NormalTok{,}\DecValTok{5}\NormalTok{,}\DecValTok{7}\NormalTok{)}
\FunctionTok{my\_mean}\NormalTok{(b) }\CommentTok{\# our own implementation}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 4.667
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{mean}\NormalTok{(b) }\CommentTok{\# the built\_in function}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 4.667
\end{verbatim}

\hypertarget{data}{%
\chapter{Data}\label{data}}

To better understand the role of data in today's economy and society, we study the usage forms and purposes of data records in human history. In the second step, we look at how a computer processes digital data.

Throughout human history, the recording and storage of data have primarily been motivated by measuring, quantifying, and keeping records of both our social and natural environments. Early on, data recording was related to economic activity and scientific endeavors. The neolithic transition from hunter-gatherer societies to agriculture and settlements (the economic development sometimes referred to as the `first industrial revolution') came along with a division of labor and more complex organizational structures of society. Because of the change in agricultural food production, more people could be fed. But it also implied that food production would need to follow a more careful planning (e.g., the right time to seed and harvest) and that the produced food (e.g., grains) would partly be stored and not consumed entirely on the spot. It is believed that partly due to these two practical problems, keeping track of time and keeping record of production quantities, neolithic societies started to use signs (numbers/letters) carved in stone or wood \citep{hogben_1983}. Keeping record of the time and later measuring and keeping record of production quantities in order to store and trade likely led to the first `data sets'. At the same time the development of mathematics, particularly geometry, took shape.

\hypertarget{processing-data-simple-calculations-in-numeral-systems}{%
\section{Processing data: simple calculations in numeral systems}\label{processing-data-simple-calculations-in-numeral-systems}}

In order to keep track of calculations with large numbers, humans started to use mechanical aids such as pebbles or shells and later developed the \href{https://en.wikipedia.org/wiki/Abacus}{counting frame (`abacus')}.\footnote{See \citet{hogben_1983}, Chapter 1 for a detailed account of the abacus' origin and usage.} We can understand the counting frame as a simple mechanical tool to process numbers (data). In order to use the abacus properly, one must agree on a standard regarding what each column of the frame represents, as well as how many beads each column can contain. In other words, one has to define what the \emph{base} of the frame's numeral system is. For example, the Roman numeral system is essentially of base 10, with `I' representing one, `X' representing ten, `C' representing one hundred, and `M' representing one thousand. Figure \ref{fig:abacus} illustrates how a counting frame based on this numeral system works (examples are written out in Arabic numbers). The first column on the right represents units of \(10^0=1\), the second \(10^1=10\), and so forth.

\begin{figure}

{\centering \includegraphics[width=0.8\linewidth]{img/abacus_own} 

}

\caption{A simple abacus.}\label{fig:abacus}
\end{figure}



From inspecting Figure \ref{fig:abacus} we recognize that the columns of the abacus are the positions of the digits, which also denote the power of 10 with which the digit is multiplied: \(139=(1 \times 10^{2}) + (3 \times 10^{1}) + (9 \times 10^{0})\). In addition, a base of 10 means that the system is based on 10 different signs (0, 1, \ldots, 9) with which we distinguish one-digit numbers. Further, we recognize that each column in the abacus has 9 beads (when we agree on the standard that a column without any bead represents the sign 0).

The numeral system with base 10 (the `decimal system') is what we use to measure/count/quantify things in our everyday life. Moreover, it is what we normally work with in applied math and statistics. However, the simple but very useful concept of the counting frame also works well for numeral systems with other bases. Historically, numeral systems with other bases existed in various cultures. For example, ancient cultures in Mesopotamia used different forms of a sexagesimal system (base 60), consisting of 60 different signs to distinguish `one-digit' numbers.

Note that this logic holds both ways: if a numeral system only consists of two different signs, it follows that the system has to be of base 2 (i.e., a `binary system'). As it turns out, this is exactly the kind of numeral system that becomes relevant once we use electronic tools, that is, digital computers, to calculate and process data.

\hypertarget{the-binary-system}{%
\subsection{The binary system}\label{the-binary-system}}

Anything related to electronically processing (and storing) \emph{digital data} has to be built on a binary system. The reason is that a microprocessor (similar to a light switch) can only represent two signs (states): \emph{on and off}. We usually refer to `off' with the sign `0' and to `on' with the sign `1'. A numeral system consisting only of the signs 0 and 1 must, thus, be of base 2. It follows that an abacus of this binary system has columns \(2^0=1\), \(2^1=2\), \(2^3=4\), and so forth. Moreover, each column has only one bead (1) or none (0). Nevertheless, we can express all (natural) numbers from the decimal system.\footnote{Representing fractions is much harder in a binary system than representing natural numbers. The reason is fractions such as \(1/3=0.333..\) actually constitute an infinite sequence of 0s and 1s. The representation of such numbers in a computer (via so-called `floating point' numbers) is thus in reality not 100\% accurate.} For example, the number 139 which we have expressed with the `decimal abacus', would be expressed as follows with a base 2 system:

\[(1 \times 2^7) + (1 \times 2^3) + (1 \times 2^1) + (1 \times 2^0) = 139.\]

More precisely, when including all columns of our binary system abacus (that is, including also the columns set to 0), we would get the following:

\[(1 \times 2^7) + (0 \times 2^6) +  (0 \times 2^5) +  (0 \times 2^4) + (1 \times 2^3) + (0 \times 2^2) + (1 \times 2^1) +  (1 \times 2^0)  = 139.\]

Now, compare this with the abacus in the decimal system above. There, we set the third column to 1, the second column to 3, and the first column (from the right) to 9, resulting in 139. If we do the same in the binary system (where we can set each column either to 0 or 1), we get 10001011. That is, the number 139 in the decimal system corresponds to 10001011 in the binary system.

How can a computer know that? To correctly print the three symbols \texttt{139} to our computer screen when dealing with the binary expression \texttt{10001011}, the computer needs to rely on a predefined mapping between binary coded values and the symbols for (Arabic) decimal numbers like \texttt{3}. Since a computer can only understand binary expressions, we have to define a \emph{standard} of how \texttt{0}s and \texttt{1}s correspond to symbols, colors, etc. that we see on the screen.

Of course, this does not change the fact that any digital data processing is, in the end, happening in the binary system. But in order to avoid having to work with a keyboard consisting only of an on/off (1/0) switch, low-level standards that define how symbols like \texttt{3}, \texttt{A}, \texttt{\#}, etc. corresponding expressions in 0s and 1s help us to interact with the computer. It makes it easier to enter data and commands into the computer as well as understand the output (i.e., the result of a calculation performed on the computer) on the screen. A standard defining how our number symbols in the decimal system correspond to binary numbers (again, reflecting the idea of an abacus) can be illustrated in the following table:

\begin{longtable}[]{@{}lllllllll@{}}
\toprule
Number & 128 & 64 & 32 & 16 & 8 & 4 & 2 & 1 \\
\midrule
\endhead
0 = & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 \\
1 = & 0 & 0 & 0 & 0 & 0 & 0 & 0 & 1 \\
2 = & 0 & 0 & 0 & 0 & 0 & 0 & 1 & 0 \\
3 = & 0 & 0 & 0 & 0 & 0 & 0 & 1 & 1 \\
\ldots{} & & & & & & & & \\
139 = & 1 & 0 & 0 & 0 & 1 & 0 & 1 & 1 \\
\bottomrule
\end{longtable}

\hypertarget{the-hexadecimal-system}{%
\subsection{The hexadecimal system}\label{the-hexadecimal-system}}

From the above table, we also recognize that binary numbers can become quite long rather quickly (have many digits). In Computer Science it is, therefore, quite common to use another numeral system to refer to binary numbers: the \emph{hexadecimal} system. In this system, we have 16 symbols available, consisting of \texttt{0}-\texttt{9} (used like in the decimal system) and \texttt{A}-\texttt{F} (for the numbers 10 to 15). Because we have 16 symbols available, each digit represents an increasing power of 16 (\(16^{0}\), \(16^{1}\), etc.). The decimal number 139 is expressed in the hexadecimal system as follows.

\[(8\times 16^1) +  (11\times 16^0) = 139.\]
More precisely, following the convention of the hexadecimal system used in Computer Science (where \texttt{B} stands for \texttt{11}), it is:
\[(8\times 16^1) +  (B\times 16^0) = 8B = 139.\]

Hence, \texttt{10001011} in the binary system is \texttt{8B} in the hexadecimal system and \texttt{139} in the decimal system. The primary use of hexadecimal notation when dealing with binary numbers is the more `human-friendly' representation of binary-coded values. First, it is shorter than the raw binary representation (as well as the decimal representation). Second, with a little bit of practice it is much easier for humans to translate forth and back between hexadecimal and binary notation than it is between decimal and binary. This is because each hexadecimal digit can directly be converted into its four-digit binary equivalent (from looking at the table above): \(8=1000\), \(B=11=1011\), thus \texttt{8B} in hexadecimal notation corresponds to \texttt{10001011} (\texttt{1000\ 1011}) in binary coded values.

\hypertarget{character-encoding}{%
\section{Character encoding}\label{character-encoding}}

Computers are not only used to process numbers but in a wide array of applications, they are used to process text. Most fundamentally, when writing computer code, we type in commands in the form of text consisting of common alphabetical letters. How can a computer understand text if it only understands \texttt{0}s and \texttt{1}s? Well, again, we have to agree on a certain standard of how \texttt{0}s and \texttt{1}s correspond to characters/letters. While the conversions of integers between different numeral systems follow all essentially the same principle (\textasciitilde{} the idea of a `digital' abacus), the introduction of standards defining how \texttt{0}s and \texttt{1}s correspond to specific letters of different human languages is way more arbitrary.

Today, many standards define how computers translate \texttt{0}s and \texttt{1}s into more meaningful symbols like numbers, letters, and special characters in various natural (human) languages. These standards are called \emph{character encodings}. They consist of what is called a `coded character set', basically a mapping of unique numbers (in the end in binary coded values) to each character in the set. For example, the classical ASCII (American Standard Code for Information Interchange) assigns the following numbers to the respective characters:

\begin{longtable}[]{@{}llll@{}}
\toprule
Binary & Hexadecimal & Decimal & Character \\
\midrule
\endhead
0011 1111 & 3F & 63 & \texttt{?} \\
0100 0001 & 41 & 65 & \texttt{A} \\
0110 0010 & 62 & 98 & \texttt{b} \\
\bottomrule
\end{longtable}

The convention that \texttt{0100\ 0001} corresponds to \texttt{A} is only true by definition of the ASCII character encoding.\footnote{Each character digit is expressed in 1 byte (8 \texttt{0}/\texttt{1} digits). The ASCII character set thus consists of \(2^{8}=256\) different characters.} It does not follow any law of nature or fundamental logic. Somebody simply defined this standard at some point. To have such standards (and widely accept them) is paramount to have functioning software and meaningful data sets that can be used and shared across many users and computers. If we write a text and store it in a file based on the ASCII standard, and that text (which, under the hood, only consists of \texttt{0}s and \texttt{1}s) is read on another computer only capable of mapping the binary code to another character set, the output would likely be complete gibberish.

In practice, the software we use to write emails, browse the Web, or conduct statistical analyses all have some of these standards built into them. Usually, we do not have to worry about encoding issues when simply interacting with a computer through such programs. The practical relevance of such standards becomes much more relevant once we \emph{store} or \emph{read} previously stored data/computer code.

\hypertarget{computer-code-and-text-files}{%
\section{Computer code and text files}\label{computer-code-and-text-files}}

Understanding the fundamentals of what digital data is and how computers process them is the basis for approaching two core themes of this book: \((I)\) How \emph{data} can be \emph{stored} digitally and be read by/imported to a computer (this will be the main subject of the next chapter), and \((II)\) how we can give instructions to a computer by writing \emph{computer code} (this will be the main topic of the first two exercises/workshops).

In both of these domains, we mainly work with one simple type of document: \emph{text files}. Text files are a collection of characters (with a given character encoding). Following the logic of how binary code is translated forth and back into text, they are a straightforward representation of the underlying information (\texttt{0}s and \texttt{1}s). They are used to store both structured data (e.g., tables), unstructured data (e.g., plain texts), or semi-structured data (such as websites). Similarly, they are used to write and store a collection of instructions to the computer (i.e., computer code) in any computer language.

From our everyday use of computers (notebooks, smartphones, etc.), we are accustomed to certain software packages to write text. Most prominently, we use programs like Microsoft Word or email clients (Outlook, Thunderbird, etc.). However, these programs are a poor choice for writing/editing plain text. Such programs tend to add all kinds of formatting to the text and use specific file formats to store formatting information in addition to the raw text. In other words, when using this type of software, we actually do not only write plain text. However, any program to read data or execute computer code expects only plain text. Therefore, we must only use \emph{text editors} to write and edit text files. For example, with \href{https://atom.io/}{Atom} or \href{https://www.rstudio.com/products/RStudio/}{RStudio}.

\hypertarget{data-processing-basics}{%
\section{Data processing basics}\label{data-processing-basics}}

By now, we already have a much better understanding of what the \texttt{0}s and \texttt{1}s stand for, how they might enter the computer, where they might be stored on the computer, and where we might see the output of processing data on a computer.

\begin{figure}

{\centering \includegraphics[width=0.75\linewidth]{img/cpu_blackbox_white} 

}

\caption{The `blackbox' of data processing.}\label{fig:dsdiagram}
\end{figure}



This section briefly summarizes the key components of processing data with a computer, following \citet{murrell_2009} (chapter 9, pages 199-204).

\hypertarget{components-of-a-standard-computing-environment}{%
\subsection{Components of a standard computing environment}\label{components-of-a-standard-computing-environment}}

Figure \ref{fig:components} illustrates the key components of a standard computing environment (PC, notebook, tablet, etc.) with which we can process data. A programming language (such as R) allows us to work with these hardware components (i.e., control them) in order to perform our tasks (e.g., cleaning/analyzing data).

\begin{figure}

{\centering \includegraphics[width=0.6\linewidth]{img/script-hardware_w} 

}

\caption{Basic components of a standard computing environment.}\label{fig:components}
\end{figure}



\begin{itemize}
\item
  The component actually \emph{processing} data is the Central Processing Unit (CPU). When using R to process data, R commands are translated into complex combinations of a small set of basic operations which the \emph{CPU} then executes.
\item
  To work with data (e.g., in R), it first must be loaded into the \emph{memory} of our computer. More specifically, into the Random Access Memory (\emph{RAM}). Typically, data is only loaded in the RAM as long as we work with it.
\item
  The \emph{Keyboard} is the typical \emph{input} hardware. It is the key tool we need to interact with the computer in this book (in contrast, in a book on graphic design this would rather be the mouse).
\item
  \emph{Mass Storage} refers to the type of computer memory we use to store data in the long run. Typically this is what we call the \emph{hard drive} or \emph{hard disk}. In these days, the relevant hard disk for storing and accessing data is actually often not the one physically built into our computer but a hard disk `in the cloud' (built into a server to which we connect over the Internet).
\item
  The \emph{Screen} is our primary output hardware that is key to interact with the computer.
\item
  Today almost all computers are connected to a local \emph{network} which in turn is connected to the Internet. With the rise of `Big Data' this component has become increasingly relevant to understand, as the key data sources might reside in this network.
\end{itemize}

Note how understanding the role of these components helps us to capture what happened in the initial example of this book (online survey of the class):

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
  You have used your computers to access a website (over the Internet) and used your keyboards to enter data into this website (a Google sheet in that case).
\item
  My R program has accessed the data you have entered into the Google sheet (again over the Internet), downloaded the data, and loaded it into the RAM on my notebook.
\item
  There, the data has been processed in order to produce an output (in the form of statistics/plots), which in the end has been shown on the screen.
\end{enumerate}

In the following, we look at a related example (taken from \citet{murrell_2009}, Chapter 9), illustrating which hardware components are involved when extracting data from a website.

\hypertarget{illustration}{%
\subsection{Illustration}\label{illustration}}

\hypertarget{downloadingaccessing-a-website}{%
\subsubsection{Downloading/accessing a website}\label{downloadingaccessing-a-website}}

First, we use our keyboard to enter the address of a website in the address bar of our browser (i.e., Firefox). The browser then accesses the network and loads the webpage to the RAM. Figure \ref{fig:htmldownload} illustrates this point.

\begin{figure}

{\centering \includegraphics[width=0.4\linewidth]{img/script-htmldownload_w} 

}

\caption{Components involved in visiting a website.}\label{fig:htmldownload}
\end{figure}



In this simple task of accessing/visiting a website, several parts of data are involved, as shown in \ref{fig:htmldownloaddata}:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\item
  The string of characters that we type into the address bar of the browser
\item
  The source code of the website (written in Hypertext markup language, HTML), being sent through the network
\item
  The HTML document (the website) stored in the RAM
\end{enumerate}

\begin{figure}

{\centering \includegraphics[width=0.4\linewidth]{img/script-htmldownloaddata_w} 

}

\caption{Data involved in visiting a website.}\label{fig:htmldownloaddata}
\end{figure}



What we just did with our browser can also be done via R. First, we tell R to download the webpage and read its source code into RAM:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{clockHTML }\OtherTok{\textless{}{-}} \FunctionTok{readLines}\NormalTok{(}\StringTok{"https://www.census.gov/popclock/"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

And we can have a look at the first lines of the web page's source code:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{head}\NormalTok{(clockHTML)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "<!DOCTYPE html>"                                                                                                                                                                                           
## [2] "<html lang=\"en\">"                                                                                                                                                                                        
## [3] "<head>"                                                                                                                                                                                                    
## [4] "\t<title>Population Clock</title>"                                                                                                                                                                         
## [5] "        <meta charset=\"UTF-8\">"                                                                                                                                                                          
## [6] "        <meta name=\"description\" content=\"Shows estimates of current USA Population overall and people by US state/county and of World Population overall, by country and most populated countries.\"/>"
\end{verbatim}

Note that what happened is essentially the same as displayed in the diagram above. The only difference is that here, we look at the raw code of the webpage, whereas in the example above, we looked at it through our browser. What the browser did, is to \emph{render} the source code, meaning it translated it into the nicely designed webpage we see in the browser window.

\hypertarget{searchingfiltering-the-webpage}{%
\subsubsection{Searching/filtering the webpage}\label{searchingfiltering-the-webpage}}

Having the webpage loaded into RAM, we can now process this data in different ways via R. For example; we can search the source code for the line that contains the part of the website with the world population counter:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{line\_number }\OtherTok{\textless{}{-}} \FunctionTok{grep}\NormalTok{(}\StringTok{\textquotesingle{}id="world{-}pop{-}container"\textquotesingle{}}\NormalTok{, clockHTML)}
\end{Highlighting}
\end{Shaded}

That is, we ask R on which line in \texttt{clockHTML} (the source code stored in RAM) the text \texttt{id="world-pop-container"} is and store the answer (the line number) in RAM under the variable name \texttt{line\_number}.

We can now check on which line the text was found.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{line\_number}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 444
\end{verbatim}

Again, we can illustrate with a simple diagram which computer components were involved in this task. First, we entered an R command with the keyboard; the CPU is processing it and as a result accesses \texttt{clockHTML} in RAM, executes the search, returns the line number, and stores it in a new variable called \texttt{line\_number} in RAM. In our current R session, we thus now have two `objects' stored in RAM: \texttt{clockHTML} and \texttt{line\_number}, which we can further process with R.

\begin{figure}

{\centering \includegraphics[width=0.4\linewidth]{img/script-ramcalc_w} 

}

\caption{Accessing data in RAM, processing it, and storing the result in RAM.}\label{fig:ramcalc}
\end{figure}



Finally, we can tell R to store the source code of the downloaded webpage on our local hard drive.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{writeLines}\NormalTok{(clockHTML, }\StringTok{"clock.html"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

That means we tell R to access the data stored in \texttt{clockHTML} in RAM and write this data to a text file called ``clock.html'' on our local hard drive (in the current working directory of our R session). The following diagram again illustrates the hardware components involved in these steps.

\begin{figure}

{\centering \includegraphics[width=0.4\linewidth]{img/script-htmlstore_w} 

}

\caption{Writing data stored in RAM to a Mass Storage device (hard drive).}\label{fig:htmlstore}
\end{figure}



\hypertarget{data-storage-and-data-structures}{%
\chapter{Data Storage and Data Structures}\label{data-storage-and-data-structures}}

A core part of the practical skills covered in this book has to do with writing, executing, and storing \emph{computer code} (i.e., instructions to a computer in a language that it understands) as well as storing and reading \emph{data}. The way data is typically stored on a computer follows quite naturally from the outlined principles of how computers process data (both technically speaking and in terms of practical applications). Based on a given standard of how \texttt{0}s and \texttt{1}s are translated into the more meaningful symbols we see on our keyboard, we simply write data (or computer code) to a \emph{text file} and save this file on the hard-disk drive of our computer. Again, what is stored on disk is in the end only consisting of \texttt{0}s and \texttt{1}s. But, given the standards outlined in the previous chapter, these \texttt{0}s and \texttt{1}s properly map to characters that we can understand when reading the file again from the disk and looking at its content on our computer screen.

\hypertarget{unstructured-data-in-text-files}{%
\section{Unstructured data in text files}\label{unstructured-data-in-text-files}}

In the simplest case, we want to store a text literally. For example, we store the following phrase

\texttt{Hello,\ World!}

in a text file named \texttt{helloworld.txt}. Technically, this means that we allocate a block of computer memory to \texttt{helloworld.txt} (a `file' is, in the end, a block of computer memory). The ending \texttt{.txt} indicates to the operating system of our computer what kind of data is stored in this file. When clicking on the file's symbol, the operating system will open the file (read it into RAM) with the default program assigned to open \texttt{.txt}-files (for example, \href{https://atom.io/}{Atom} or \href{https://www.rstudio.com/}{RStudio}). That is, the \emph{format} of a file says something about how to interpret the \texttt{0}s and \texttt{1}s. If the text editor displays \texttt{Hello\ World!} as the content of this file, the program correctly interprets the \texttt{0}s and \texttt{1}s as representing plain text and uses the correct character encoding to convert the raw \texttt{0}s and \texttt{1}s yet again to symbols that we can read more easily. Similarly, we can show the content of the file in the command-line terminal (here OSX or Linux):

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{cat}\NormalTok{ helloworld.txt}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Hello World!
\end{verbatim}

Or, from the R-console:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{system}\NormalTok{(}\StringTok{"cat helloworld.txt"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

However, we can also use a command-line program (here, \texttt{xxd}) to display the content of \texttt{helloworld.txt} without actually `translating' it into ASCII characters.\footnote{To be precise, this program shows both the raw binary content as well as its ASCII representation.} That is, we can directly look at how the content looks like as \texttt{0}s and \texttt{1}s:

\begin{Shaded}
\begin{Highlighting}[]
\ExtensionTok{xxd} \AttributeTok{{-}b}\NormalTok{ helloworld.txt}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 00000000: 01001000 01100101 01101100 01101100 01101111 00100000  Hello 
## 00000006: 01010111 01101111 01110010 01101100 01100100 00100001  World!
\end{verbatim}

Similarly, we can display the content in hexadecimal values:

\begin{Shaded}
\begin{Highlighting}[]
\ExtensionTok{xxd}\NormalTok{  helloworld.txt}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 00000000: 4865 6c6c 6f20 576f 726c 6421            Hello World!
\end{verbatim}

Next, consider the text file \texttt{hastamanana.txt}, which was written and stored with another character encoding. When looking at the content, assuming the now common UTF-8 encoding, the content seems a little odd:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{cat}\NormalTok{ hastamanana.txt}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Hasta Ma?ana!
\end{verbatim}

We see that it is a short phrase written in Spanish. Strangely it contains the character \texttt{?} in the middle of a word. The occurrence of special characters in unusual places of text files is an indication of using the wrong character encoding to display the text. Let's check what the file's encoding is.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{file} \AttributeTok{{-}b}\NormalTok{ hastamanana.txt}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## ISO-8859 text
\end{verbatim}

This tells us that the file is encoded with ISO-8859 (`Latin1'), a character set for several Latin languages (including Spanish). Knowing this, we can check how the content looks like when we change the encoding to UTF-8 (which then properly displays the content on the screen)\footnote{Note that changing the encoding in this way only works well if we know what the original encoding of the file is (i.e., the encoding used when the file was initially created). While the \texttt{file} command above simply tells us what the original encoding was, it has to make an educated guess in most cases (as the original encoding is often not stored as metadata describing a file). See, for example, the \href{https://www-archive.mozilla.org/projects/intl/chardet.html}{Mozilla Charset Detectors}.}:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{iconv} \AttributeTok{{-}f}\NormalTok{ iso{-}8859{-}1 }\AttributeTok{{-}t}\NormalTok{ utf{-}8 hastamanana.txt }\KeywordTok{|} \FunctionTok{cat}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Hasta Mañana!
\end{verbatim}

When working with data, we must therefore ensure that we apply the proper standards to translate the binary coded values into a character/text representation that is easier to understand and work with. In recent years, much more general (or even `universal') standards have been developed and adopted worldwide, particularly UTF-8. Thus, if we deal with recently generated data sets, we usually can expect that they are encoded in UTF-8, independent of the data's country of origin. However, when working on a research project with slightly older data sets from different sources, encoding issues still occur quite frequently. It is thus crucial to understand the origin of the problem early on when the data seem to display weird characters. Encoding issues are among the most basic problems that can occur when importing data.

So far, we have only looked at very simple examples of data stored in text files, essentially only containing short phrases of text. Nevertheless, the most common formats of storing and transferring data (CSV, XML, JSON, etc.) build on exactly the same principle: characters in a text-file. The difference between such formats and the simple examples above is that their content is structured in a very specific way. That is, on top of the lower-level conversion of \texttt{0}s and \texttt{1}s into characters/text, we add another standard, a data format, giving the data more \emph{structure}, making it even simpler to interpret and work with the data on a computer. Once we understand how data is represented at a low level (the topic of the previous chapter), it is much easier to understand and distinguish different forms of storing structured data.

\hypertarget{structured-data-formats}{%
\section{Structured data formats}\label{structured-data-formats}}

As nicely pointed out by \citet{murrell_2009}, the most commonly used software today is designed in a very `user-friendly' way, leading to situations in which we as users are `told' by the computer what to do (rather than the other way around). A prominent symptom of this phenomenon is that specific software is automatically assigned to open files of specific formats, so we don't have to bother about what the file's format actually is but only have to click on the icon representing the file.

However, if we actually want to engage seriously with a world driven by data, we have to go beyond the habit of simply clicking on (data-)files and let the computer choose what to do with it. Since most common formats to store data are in essence text files (in research contexts usually in ASCII or UTF-8 encoding), a good way to start engaging with a data set is to look at the raw text file containing the data in order to get an idea of how the data is structured.

For example, let's have a look at the file \texttt{ch\_gdp.csv}. When opening the file in a text editor we see the following:

\begin{verbatim}
year,gdp_chfb
1980,184
1985,244
1990,331
1995,374
2000,422
2005,464
\end{verbatim}

At first sight, the file contains a collection of numbers and characters. Having a closer look, certain structural features become apparent. The content is distributed over several rows, with each row containing a comma character (\texttt{,}). Moreover, the first row seems systematically different from the following rows: it is the only row containing alphabetical characters, all the other rows contain numbers. Rather intuitively, we recognize that the specific way in which the data in this file is structured might, in fact, represent a table with two columns: one with the variable \texttt{year} describing the year of each observation (row), the other with the variable \texttt{gdp\_chfb} with numerical values describing another characteristic of each observation/row (we can guess that this variable is somehow related to GDP and CHF/Swiss francs).

The question arises, how do we work with this data? Recall that it is simply a text file. All the information stored in it is displayed above. How could we explain to the computer that this is a table with two columns containing two variables describing six observations? We would have to come up with a sequence of instructions/rules (i.e., an algorithm) of how to distinguish columns and rows when all that a computer can do is sequentially read one character after the other (more precisely one \texttt{0}/\texttt{1} after the other, representing characters).

This algorithm could be something along the lines of:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  Start with an empty table consisting of one cell (1 row/column).
\item
  While the end of the input file is not yet reached, do the following:
  Read characters from the input file, and add them one-by-one to the current cell.
  If you encounter the character `\texttt{,}', ignore it, create a new field, and jump to the new field.
  If you encounter the end of the line, create a new row and jump to the new row.
\end{enumerate}

Consider the usefulness of this algorithm. It would certainly work quite well for the particular file we are looking at. But what if another file we want to read data from does not contain any `\texttt{,}' but only `\texttt{;}'? We would have to tweak the algorithm accordingly. Hence, again, it is extremely useful if we can agree on certain standards of how data structured as a table/matrix is stored in a text file.

\hypertarget{csvs-and-fixed-width-format}{%
\subsection{CSVs and fixed-width format}\label{csvs-and-fixed-width-format}}

Incidentally, the example we are looking at here is in line with such a standard, i.e., Comma-Separated Values (CSV, therefore \texttt{.csv}). In technical terms, the simple algorithm outlined above is a CSV \href{https://en.wikipedia.org/wiki/Parsing\#Computer_languages}{parser}. A CSV parser is a software component which takes a text file with CSV standard structure as input and builds a data structure in the form of a table (represented in RAM). If the input file does not follow the agreed-on CSV standard, the parser will likely fail to properly `parse' (read/translate) the input.

CSV files are very often used to transfer and store data in a table-like format. They essentially are based on two rules defining the structure of the data: commas delimit values/fields in a row and the end of a line indicates the end of a row.\footnote{In addition, if a field contains itself a comma (the comma is part of the data), the field needs to be surrounded by double-quotes (\texttt{"}). If a field contains double-quotes it also has to be surrounded by double-quotes, and the double quotes in the field must be preceded by an additional double-quote.}

The comma is clearly visible when looking at the raw text content of \texttt{ch\_gdp.csv}. However, how does the computer know that a line is ending? By default most programs to work with text files do not show an explicit symbol for line endings but instead display the text on the next line. Under the hood, these line endings are, however, non-printable characters. We can see this when investigating \texttt{ch\_gdp.csv} in the command-line terminal (via \texttt{xxd}):

\begin{Shaded}
\begin{Highlighting}[]
\ExtensionTok{xxd}\NormalTok{ ch\_gdp.csv}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 00000000: efbb bf79 6561 722c 6764 705f 6368 6662  ...year,gdp_chfb
## 00000010: 0d31 3938 302c 3138 340d 3139 3835 2c32  .1980,184.1985,2
## 00000020: 3434 0d31 3939 302c 3333 310d 3139 3935  44.1990,331.1995
## 00000030: 2c33 3734 0d32 3030 302c 3432 320d 3230  ,374.2000,422.20
## 00000040: 3035 2c34 3634                           05,464
\end{verbatim}

When comparing the hexadecimal values with the characters they represent on the right side, we recognize that a full stop (\texttt{.}) is printed to the output right before every year. Moreover, when inspecting the hexadecimal code, we recognize that this \texttt{.} corresponds to \texttt{0d} in the hexadecimal code. \texttt{0d} is indeed the sequence of \texttt{0}s and \texttt{1} indicating the end of a line. Because this character does not actually correspond to a symbol printed on the screen, it is replaced by \texttt{.} in the printed output of the text.\footnote{The same applies to other sequences of \texttt{0}s and \texttt{1}s that do not correspond to a printable character.}

While CSV files have become a very common way to store `flat'/table-like data in plain text files, several similar formats can be encountered in practice. Most commonly they either use a different delimiter (for example, tabs/white space) to separate fields in a row, or fields are defined to consist of a fixed number of characters (so-called fixed-width formats). In addition, various more complex standards to store and transfer digital data are widely used to store data. Particularly in the context of web data (data stored and transferred online), files storing data in these formats (e.g., XML, JSON, YAML, etc.) are in the end, just plain text files. However, they contain a larger set of special characters/delimiters (or combinations of these) to indicate the structure of the data stored in them.

\hypertarget{units-of-informationdata-storage}{%
\section{Units of information/data storage}\label{units-of-informationdata-storage}}

The question of how to store digital data on computers also raises the question of storage capacity. Because every type of digital data can, in the end, only be stored as \texttt{0}s and \texttt{1}s, it makes perfect sense to define the smallest unit of information in computing as consisting of either a \texttt{0} or a \texttt{1}. We call this basic unit a \emph{bit} (from \emph{bi}nary dig\emph{it}; abbrev. `b'). Recall that the decimal number \texttt{139} corresponds to the binary number \texttt{10001011}. To store this number on a hard disk, we require a capacity of 8 bits or one \emph{byte} (1 byte = 8 bits; abbrev. `B'). Historically, one byte encoded a single character of text (i.e., in the ASCII character encoding system). 4 bytes (or 32 bits) are called a \emph{word}. The following figure illustrates this point.

\begin{figure}

{\centering \includegraphics[width=0.4\linewidth]{img/bitbyte} 

}

\caption{Byte and bit.}\label{fig:bitbyteword}
\end{figure}



Bigger units for storage capacity usually build on bytes:

\begin{itemize}
\tightlist
\item
  \(1 \text{ kilobyte (KB)} = 1000^{1} \approx 2^{10} \text{ bytes}\)
\item
  \(1 \text{ megabyte (MB)} = 1000^{2} \approx 2^{20} \text{ bytes}\)
\item
  \(1 \text{ gigabyte (GB)} = 1000^{3} \approx 2^{30} \text{ bytes}\)
\item
  \(1 \text{ terabyte (TB)} = 1000^{4} \approx 2^{40} \text{ bytes}\)
\item
  \(1 \text{ petabyte (PB)} = 1000^{5} \approx 2^{50} \text{ bytes}\)
\item
  \(1 \text{ exabyte (EB)} = 1000^{6} \approx 2^{60} \text{ bytes}\)
\item
  \(1 \text{ zettabyte (ZB)} = 1000^{7} \approx 2^{70} \text{ bytes}\)
\end{itemize}

\(1 ZB = 1000000000000000000000\text{ bytes} = 1 \text{ billion terabytes} = 1 \text{ trillion gigabytes}.\)

\hypertarget{data-structures-and-data-types-in-r}{%
\section{Data structures and data types in R}\label{data-structures-and-data-types-in-r}}

So far, we have focused on how data is stored on the hard disk. That is, we discovered how '0's and '1's correspond to characters (using specific standards: character encodings), how a sequence of characters in a text file is a useful way to store data on a hard disk, and how specific standards are used to structure the data in a meaningful way (using special characters). When thinking about how to store data on a hard disk, we are usually concerned with how much space (in bytes) the data set needs, how well it is transferable/understandable, and how well we can retrieve information from it. All of these aspects go into the decision of what structure/format to choose etc.

However, none of these considers how we actively work with the data. For example, when we want to sum up the \texttt{gdp\_chfb} column in \texttt{ch\_gdp.csv} (the table example above), do we actually work within the CSV data structure? The answer is usually no.

Recall from the basics of data processing that data stored on the hard disk is loaded into RAM to work with (analysis, manipulation, cleaning, etc.). The question arises as to how data is structured in RAM? That is, what data structures/formats are used to work with the data actively?

We distinguish two basic characteristics:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  Data \textbf{types}: integers; real numbers (`numeric values', floating point numbers); text (`string', `character values').
\item
  Basic \textbf{data structures} in RAM:
  - \emph{Vectors}
  - \emph{Factors}
  - \emph{Arrays/matrices}
  - \emph{Lists}
  - \emph{Data frames} (very \texttt{R}-specific)
\end{enumerate}

Depending on the data structure/format stored in on the hard disk, the data will be more or less usefully represented in one of the above structures in RAM. For example, in the \texttt{R} language it is quite common to represent data stored in CSVs on disk as \emph{data frames} in RAM. Similarly, it is quite common to represent a more complex format on disk, such as JSON, as a nested list in RAM.

Importing data from the hard disk (or another mass storage device) into RAM in order to work with it in \texttt{R} essentially means reading the sequence of characters (in the end, of course, \texttt{0}s and \texttt{1}s) and mapping them, given the structure they are stored in, into one of these structures for representation in RAM.\footnote{In the chapter on data gathering and data import, we will cover this crucial step in detail.}.

\hypertarget{data-types}{%
\subsection{Data types}\label{data-types}}

From the previous chapters, we know that digital data (\texttt{0}s and \texttt{1}s) can be interpreted in different ways depending on the \emph{format} it is stored in. Similarly, data loaded into RAM can be interpreted differently by R depending on the data \emph{type}. Some operators or functions in R only accept data of a specific type as arguments. For example, we can store the numeric values \texttt{1.5} and \texttt{3} in the variables \texttt{a} and \texttt{b}, respectively.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{a }\OtherTok{\textless{}{-}} \FloatTok{1.5}
\NormalTok{b }\OtherTok{\textless{}{-}} \DecValTok{3}
\end{Highlighting}
\end{Shaded}

R interprets this data as type \texttt{double} (class `numeric'; a `double precision floating point number'):

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{typeof}\NormalTok{(a)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "double"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{class}\NormalTok{(a)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "numeric"
\end{verbatim}

Given that these bytes of data are interpreted as numeric, we can use operators (here: math operators) that can work with such types:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{a }\SpecialCharTok{+}\NormalTok{ b}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 4.5
\end{verbatim}

If we define \texttt{a} and \texttt{b} as follows, R will interpret the values stored in \texttt{a} and \texttt{b} as text (\texttt{character}).

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{a }\OtherTok{\textless{}{-}} \StringTok{"1.5"}
\NormalTok{b }\OtherTok{\textless{}{-}} \StringTok{"3"}
\end{Highlighting}
\end{Shaded}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{typeof}\NormalTok{(a)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "character"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{class}\NormalTok{(a)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "character"
\end{verbatim}

Now the same line of code as above will result in an error:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{a }\SpecialCharTok{+}\NormalTok{ b}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Error in a + b: non-numeric argument to binary operator
\end{verbatim}

The reason is that an operator \texttt{+} expects numeric or integer values as arguments. When importing data sets with many different variables (columns), it is thus necessary to ensure that each column is interpreted in the intended way. That is, we have to make sure R is assigning the right type to each of the imported variables. Usually, this is done automatically. However, with large and complex data sets, the automatic recognition of data types when importing data can fail when the data is not perfectly cleaned/prepared (in practice, this is very often the case).

\hypertarget{data-structures}{%
\subsection{Data structures}\label{data-structures}}

For now, we have only looked at individual bytes of data. An entire data set can consist of gigabytes of data containing text and numeric values. How can such collections of data values be represented in R? Here, we look at the main data structures implemented in R. All of these structures will play a role in some of the hands-on exercises.

\hypertarget{vectors}{%
\subsubsection{Vectors}\label{vectors}}

Vectors are collections of values of the same type. They can contain either all numeric values or all character values. We will use the following symbol to refer to vectors.

\begin{figure}

{\centering \includegraphics[width=0.05\linewidth]{img/numvec} 

}

\caption{Illustration of a numeric vector.}\label{fig:numvec}
\end{figure}



For example, we can initiate a character vector containing the names of persons:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{persons }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\StringTok{"Andy"}\NormalTok{, }\StringTok{"Brian"}\NormalTok{, }\StringTok{"Claire"}\NormalTok{)}
\NormalTok{persons}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "Andy"   "Brian"  "Claire"
\end{verbatim}

And we can initiate a numeric vector with the age of these persons:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{ages }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\DecValTok{24}\NormalTok{, }\DecValTok{50}\NormalTok{, }\DecValTok{30}\NormalTok{)}
\NormalTok{ages}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 24 50 30
\end{verbatim}

\hypertarget{factors}{%
\subsubsection{Factors}\label{factors}}

Factors are sets of categories. Thus, the values come from a fixed set of possible values.

\begin{figure}

{\centering \includegraphics[width=0.05\linewidth]{img/factor} 

}

\caption{Illustration of a factor.}\label{fig:factor}
\end{figure}



For example, we might want to initiate a factor that indicates the gender of a number of people.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{gender }\OtherTok{\textless{}{-}} \FunctionTok{factor}\NormalTok{(}\FunctionTok{c}\NormalTok{(}\StringTok{"Male"}\NormalTok{, }\StringTok{"Male"}\NormalTok{, }\StringTok{"Female"}\NormalTok{))}
\NormalTok{gender}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] Male   Male   Female
## Levels: Female Male
\end{verbatim}

\hypertarget{matricesarrays}{%
\subsubsection{Matrices/Arrays}\label{matricesarrays}}

Matrices are two-dimensional collections of values, arrays of higher-dimensional collections of values of the same type. For an illustration, consider the following \(3\times3\) integer matrix and the \(3\times3\times3\) integer array.

\begin{figure}

{\centering \includegraphics[width=0.15\linewidth]{img/matrix} 

}

\caption{Illustration of an integer matrix.}\label{fig:matrix}
\end{figure}



\begin{figure}

{\centering \includegraphics[width=0.25\linewidth]{img/array} 

}

\caption{Illustration of three-dimensional integer array.}\label{fig:array}
\end{figure}



In R, we can, for example, initiate a three-row/three-column integer matrix as follows.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{my\_matrix }\OtherTok{\textless{}{-}} \FunctionTok{matrix}\NormalTok{(}\DecValTok{1}\SpecialCharTok{:}\DecValTok{9}\NormalTok{, }\AttributeTok{nrow =} \DecValTok{3}\NormalTok{)}
\NormalTok{my\_matrix}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##      [,1] [,2] [,3]
## [1,]    1    4    7
## [2,]    2    5    8
## [3,]    3    6    9
\end{verbatim}

Similarly, we can initiate a three-dimensional array via the \texttt{array()}-function as shown below. Note that instead of defining only the numbers of rows (columns), you need to define the number of elements in each dimension via the \texttt{dim}-parameter.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{my\_array }\OtherTok{\textless{}{-}} \FunctionTok{array}\NormalTok{(}\DecValTok{1}\SpecialCharTok{:}\DecValTok{27}\NormalTok{, }\AttributeTok{dim =} \FunctionTok{c}\NormalTok{(}\DecValTok{3}\NormalTok{,}\DecValTok{3}\NormalTok{,}\DecValTok{3}\NormalTok{))}
\NormalTok{my\_array}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## , , 1
## 
##      [,1] [,2] [,3]
## [1,]    1    4    7
## [2,]    2    5    8
## [3,]    3    6    9
## 
## , , 2
## 
##      [,1] [,2] [,3]
## [1,]   10   13   16
## [2,]   11   14   17
## [3,]   12   15   18
## 
## , , 3
## 
##      [,1] [,2] [,3]
## [1,]   19   22   25
## [2,]   20   23   26
## [3,]   21   24   27
\end{verbatim}

We can access parts of matrices and arrays via indexing implemented with the \texttt{{[}{]}}-syntax. For example, select a particular row, column, or individual element of a matrix.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Select the first row}
\NormalTok{my\_matrix[}\DecValTok{1}\NormalTok{,]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 1 4 7
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Select the second column}
\NormalTok{my\_matrix[,}\DecValTok{2}\NormalTok{]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 4 5 6
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Select the value in the second column, third row.}
\NormalTok{my\_matrix[}\DecValTok{3}\NormalTok{,}\DecValTok{2}\NormalTok{]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 6
\end{verbatim}

Following the same syntax, we can access specific parts of an array. Note, though, that we need to consider the correct number of dimensions of the corresponding array object. In the example of above, we have generated three-dimensional array, hence there needs to be three parts within the \texttt{{[}{]}}.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Select the first rows}
\NormalTok{my\_array[}\DecValTok{1}\NormalTok{,,]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##      [,1] [,2] [,3]
## [1,]    1   10   19
## [2,]    4   13   22
## [3,]    7   16   25
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Select the "third matrix"}
\NormalTok{my\_array[,,}\DecValTok{3}\NormalTok{]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##      [,1] [,2] [,3]
## [1,]   19   22   25
## [2,]   20   23   26
## [3,]   21   24   27
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Select the "last" element in the array}
\NormalTok{my\_array[}\DecValTok{3}\NormalTok{,}\DecValTok{3}\NormalTok{,}\DecValTok{3}\NormalTok{]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 27
\end{verbatim}

\hypertarget{data-frames-tibbles-and-data-tables}{%
\subsubsection{Data frames, tibbles, and data tables}\label{data-frames-tibbles-and-data-tables}}

Data frames are the typical representation of a (table-like) data set in R. Each column can contain a vector of a given data type (or a factor), but all columns need to be of identical length. Thus in the context of data analysis, we would say that each row of a data frame contains an observation, and each column contains a characteristic of this observation.

\begin{figure}

{\centering \includegraphics[width=0.2\linewidth]{img/dataframe} 

}

\caption{Illustration of a data frame.}\label{fig:df}
\end{figure}



The historical implementation of data frames in R is not very appropriate to work with large datasets. \footnote{In the early days of R, this was not an issue because datasets that are rather large by today's standards (in the Gigabytes) could not have been handled properly by normal computers anyway (due to a lack of RAM).} These days there are new implementations of the data frame concept in R provided by different packages, which aim at making data processing based on `data frames' faster. One is called \texttt{tibbles}, implemented and used in the \texttt{tidyverse} packages. The other is called \texttt{data\ table}, implemented in the \texttt{data.\ table}-package. In this book, we will encounter primarily classical \texttt{data.\ frame} and \texttt{tibble} (however, there will also be some hints to tutorials with \texttt{data.\ table}). In any case, once we understand data frames in general, working with \texttt{tibble} and \texttt{data.\ table} is quite easy because functions that accept classical data frames as arguments also accept those newer implementations.

Here is how we define a data frame in R, based on the examples of vectors and factors shown above.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{df }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{person =}\NormalTok{ persons, }\AttributeTok{age =}\NormalTok{ ages, }\AttributeTok{gender =}\NormalTok{ gender)}
\NormalTok{df}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   person age gender
## 1   Andy  24   Male
## 2  Brian  50   Male
## 3 Claire  30 Female
\end{verbatim}

\hypertarget{lists}{%
\subsubsection{Lists}\label{lists}}

Unlike data frames, lists can contain different data types in each element. Moreover, they even can contain different data structures of \emph{different dimensions} in each element. For example, a list could contain different other lists, data frames, and vectors with differing numbers of elements.

\begin{figure}

{\centering \includegraphics[width=0.2\linewidth]{img/list} 

}

\caption{Illustration of a list.}\label{fig:list}
\end{figure}



This flexibility can easily be demonstrated by combining some of the data structures created in the examples above:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{my\_list }\OtherTok{\textless{}{-}} \FunctionTok{list}\NormalTok{(my\_array, my\_matrix, df)}
\NormalTok{my\_list}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [[1]]
## , , 1
## 
##      [,1] [,2] [,3]
## [1,]    1    4    7
## [2,]    2    5    8
## [3,]    3    6    9
## 
## , , 2
## 
##      [,1] [,2] [,3]
## [1,]   10   13   16
## [2,]   11   14   17
## [3,]   12   15   18
## 
## , , 3
## 
##      [,1] [,2] [,3]
## [1,]   19   22   25
## [2,]   20   23   26
## [3,]   21   24   27
## 
## 
## [[2]]
##      [,1] [,2] [,3]
## [1,]    1    4    7
## [2,]    2    5    8
## [3,]    3    6    9
## 
## [[3]]
##   person age gender
## 1   Andy  24   Male
## 2  Brian  50   Male
## 3 Claire  30 Female
\end{verbatim}

\hypertarget{high-dimensional-data}{%
\chapter{High-Dimensional Data}\label{high-dimensional-data}}

So far, we have only looked at data structured in a flat/table-like representation (e.g.~CSV files). In applied econometrics/statistics, it is common to only work with data sets stored in such format. Data manipulation, filtering, aggregation, etc. presupposes data in a table-like format. Hence, storing data in this format makes perfect sense.

As we observed in the previous chapter, the CSV structure has some disadvantages when representing more complex data in a text file. This is in particular true if the data contains nested observations (i.e., hierarchical structures). While a representation in a CSV file is theoretically possible, it is often far from practical to use other formats for such data. On the one hand, it is likely less intuitive to read the data correctly. On the other hand, storing the data in a CSV file might introduce a lot of redundancy. That is, the identical values of some variables would have to be repeated in the same column. The following code block illustrates this point for a data set on two families (\citep{murrell_2009}, p.~116).

\begin{verbatim}
father mother  name     age  gender
               John      33  male
               Julia     32  female
John   Julia   Jack       6  male
John   Julia   Jill       4  female
John   Julia   John jnr   2  male
               David     45  male
               Debbie    42  female
David  Debbie  Donald    16  male
David  Debbie  Dianne    12  female
\end{verbatim}

From simply looking at the data, we can make the best guess which observations belong together (are one family). However, the implied hierarchy is not apparent at first sight. While it might not matter too much that several values have to be repeated several times in this format, given that this data set is so small, the repeated values can become a problem when the data set is much larger. For each time \texttt{John} is repeated in the \texttt{father}column, we use 4 bytes of memory. Suppose millions of people are in this data set, and we have to transfer this data set very often over a computer network. In that case, these repetitions can become quite costly (as we would need more storage capacity and network resources).

Issues with complex/hierarchical data (with several observation types), intuitive human readability (self-describing), and efficiency in storage, as well as transfer, are all of great importance on the Web. In the context of web technologies, several data formats have been put forward to address these issues. Here, we discuss the two most prominent of these formats: \href{https://en.wikipedia.org/wiki/XML}{Extensible Markup Language (XML)} and \href{https://en.wikipedia.org/wiki/JSON}{JavaScript Object Notation (JSON)}.

\hypertarget{deciphering-xml}{%
\section{Deciphering XML}\label{deciphering-xml}}

Before going into more technical details, let's try to figure out the basic logic behind the XML format by simply looking at some raw example data. For this, we turn to the Point Nemo case study in \citep{murrell_2009}. The following code block shows the upper part of the data set downloaded from NASA's LAS server (here in a CSV-type format).

\begin{verbatim}
             VARIABLE : Monthly Surface Clear-sky Temperature (ISCCP) (Celsius)
             FILENAME : ISCCPMonthly_avg.nc
             FILEPATH : /usr/local/fer_data/data/
             BAD FLAG : -1.E+34       
             SUBSET   : 48 points (TIME)
             LONGITUDE: 123.8W(-123.8)
             LATITUDE : 48.8S
             123.8W
16-JAN-1994 00    9.200012
16-FEB-1994 00    10.70001
16-MAR-1994 00    7.5
16-APR-1994 00    8.100006
\end{verbatim}

Below, the same data is now displayed in XML format. Note that in both cases, the data is simply stored in a text file. However, it is stored in a format that imposes a different \emph{structure} on the data.

\begin{verbatim}
<?xml version="1.0"?>
<temperatures>
<variable>Monthly Surface Clear-sky Temperature (ISCCP) (Celsius)</variable>
<filename>ISCCPMonthly_avg.nc</filename>
<filepath>/usr/local/fer_data/data/</filepath>
<badflag>-1.E+34</badflag>

<subset>48 points (TIME)</subset>
<longitude>123.8W(-123.8)</longitude>
<latitude>48.8S</latitude>
<case date="16-JAN-1994" temperature="9.200012" />
<case date="16-FEB-1994" temperature="10.70001" />
<case date="16-MAR-1994" temperature="7.5" />
<case date="16-APR-1994" temperature="8.100006" />

...
</temperatures>
\end{verbatim}

What features does the format have? What is its logic? Is there room for improvement? The XML data structure becomes more apparent by using indentation and code highlighting.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{\textless{}?xml}\OtherTok{ version=}\StringTok{"1.0"}\FunctionTok{?\textgreater{}}
\NormalTok{  \textless{}}\KeywordTok{temperatures}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{variable}\NormalTok{\textgreater{}Monthly Surface Clear{-}sky Temperature (ISCCP) (Celsius)\textless{}/}\KeywordTok{variable}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{filename}\NormalTok{\textgreater{}ISCCPMonthly\_avg.nc\textless{}/}\KeywordTok{filename}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{filepath}\NormalTok{\textgreater{}/usr/local/fer\_data/data/\textless{}/}\KeywordTok{filepath}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{badflag}\NormalTok{\textgreater{}{-}1.E+34\textless{}/}\KeywordTok{badflag}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{subset}\NormalTok{\textgreater{}48 points (TIME)\textless{}/}\KeywordTok{subset}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{longitude}\NormalTok{\textgreater{}123.8W({-}123.8)\textless{}/}\KeywordTok{longitude}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{latitude}\NormalTok{\textgreater{}48.8S\textless{}/}\KeywordTok{latitude}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}JAN{-}1994"}\OtherTok{ temperature=}\StringTok{"9.200012"}\NormalTok{ /\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}FEB{-}1994"}\OtherTok{ temperature=}\StringTok{"10.70001"}\NormalTok{ /\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}MAR{-}1994"}\OtherTok{ temperature=}\StringTok{"7.5"}\NormalTok{ /\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}APR{-}1994"}\OtherTok{ temperature=}\StringTok{"8.100006"}\NormalTok{ /\textgreater{}}

\NormalTok{...}
\NormalTok{  \textless{}/}\KeywordTok{temperatures}\NormalTok{\textgreater{}}
\end{Highlighting}
\end{Shaded}

First, note how special characters are used to define the document's structure. We notice that \texttt{\textless{}} and \texttt{\textgreater{}}, containing some text labels seem to play a key role in defining the structure. These building blocks are called `XML tags'. We are free to choose what tags we want to use. In essence, we can define them ourselves to most properly describe the data. Moreover, the example data reveals the flexibility of XML to depict hierarchical structures.

The actual content we know from the CSV-type example above is nested between the `\texttt{temperatures}'-tags, indicating what the data is about.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{  \textless{}}\KeywordTok{temperatures}\NormalTok{\textgreater{}}
\NormalTok{...}
\NormalTok{  \textless{}/}\KeywordTok{temperatures}\NormalTok{\textgreater{}}
\end{Highlighting}
\end{Shaded}

Comparing the actual content between these tags with the CSV-type format above, we further recognize that there are two principal ways to link variable names to values.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{    \textless{}}\KeywordTok{variable}\NormalTok{\textgreater{}Monthly Surface Clear{-}sky Temperature (ISCCP) (Celsius)\textless{}/}\KeywordTok{variable}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{filename}\NormalTok{\textgreater{}ISCCPMonthly\_avg.nc\textless{}/}\KeywordTok{filename}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{filepath}\NormalTok{\textgreater{}/usr/local/fer\_data/data/\textless{}/}\KeywordTok{filepath}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{badflag}\NormalTok{\textgreater{}{-}1.E+34\textless{}/}\KeywordTok{badflag}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{subset}\NormalTok{\textgreater{}48 points (TIME)\textless{}/}\KeywordTok{subset}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{longitude}\NormalTok{\textgreater{}123.8W({-}123.8)\textless{}/}\KeywordTok{longitude}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{latitude}\NormalTok{\textgreater{}48.8S\textless{}/}\KeywordTok{latitude}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}JAN{-}1994"}\OtherTok{ temperature=}\StringTok{"9.200012"}\NormalTok{ /\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}FEB{-}1994"}\OtherTok{ temperature=}\StringTok{"10.70001"}\NormalTok{ /\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}MAR{-}1994"}\OtherTok{ temperature=}\StringTok{"7.5"}\NormalTok{ /\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}APR{-}1994"}\OtherTok{ temperature=}\StringTok{"8.100006"}\NormalTok{ /\textgreater{}}
\end{Highlighting}
\end{Shaded}

One way is to define opening and closing XML-tags with the variable name and surround the value with them, such as in \texttt{\textless{}filename\textgreater{}ISCCPMonthly\_avg.nc\textless{}/filename\textgreater{}}. Another way would be to encapsulate the values within one tag by defining tag attributes such as in \texttt{\textless{}case\ date="16-JAN-1994"\ temperature="9.200012"\ /\textgreater{}}. In many situations, both approaches can make sense. For example, the way the temperature measurements are encoded in the example data set is based on the tag-attributes approach:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}JAN{-}1994"}\OtherTok{ temperature=}\StringTok{"9.200012"}\NormalTok{ /\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}FEB{-}1994"}\OtherTok{ temperature=}\StringTok{"10.70001"}\NormalTok{ /\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}MAR{-}1994"}\OtherTok{ temperature=}\StringTok{"7.5"}\NormalTok{ /\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\OtherTok{ date=}\StringTok{"16{-}APR{-}1994"}\OtherTok{ temperature=}\StringTok{"8.100006"}\NormalTok{ /\textgreater{}}
\end{Highlighting}
\end{Shaded}

We could rewrite this by only using XML tags and no attributes:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{  \textless{}}\KeywordTok{cases}\NormalTok{\textgreater{}    }
\NormalTok{    \textless{}}\KeywordTok{case}\NormalTok{\textgreater{}}
\NormalTok{      \textless{}}\KeywordTok{date}\NormalTok{\textgreater{}16{-}JAN{-}1994\textless{}}\KeywordTok{date}\NormalTok{/\textgreater{}}
\NormalTok{      \textless{}}\KeywordTok{temperature}\NormalTok{\textgreater{}9.200012\textless{}}\KeywordTok{temperature}\NormalTok{/\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\NormalTok{/\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\NormalTok{\textgreater{}}
\NormalTok{      \textless{}}\KeywordTok{date}\NormalTok{\textgreater{}16{-}FEB{-}1994\textless{}}\KeywordTok{date}\NormalTok{/\textgreater{}}
\NormalTok{      \textless{}}\KeywordTok{temperature}\NormalTok{\textgreater{}10.70001\textless{}}\KeywordTok{temperature}\NormalTok{/\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\NormalTok{/\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\NormalTok{\textgreater{}}
\NormalTok{      \textless{}}\KeywordTok{date}\NormalTok{\textgreater{}16{-}MAR{-}1994\textless{}}\KeywordTok{date}\NormalTok{/\textgreater{}}
\NormalTok{      \textless{}}\KeywordTok{temperature}\NormalTok{\textgreater{}7.5\textless{}}\KeywordTok{temperature}\NormalTok{/\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\NormalTok{/\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\NormalTok{\textgreater{}}
\NormalTok{      \textless{}}\KeywordTok{date}\NormalTok{\textgreater{}16{-}APR{-}1994\textless{}}\KeywordTok{date}\NormalTok{/\textgreater{}}
\NormalTok{      \textless{}}\KeywordTok{temperature}\NormalTok{\textgreater{}8.100006\textless{}}\KeywordTok{temperature}\NormalTok{/\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{case}\NormalTok{/\textgreater{}}
\NormalTok{  \textless{}}\KeywordTok{cases}\NormalTok{/\textgreater{}}
\end{Highlighting}
\end{Shaded}

As long as we follow the basic XML syntax, both versions are valid, and XML parsers can read them equally well.

Note the key differences between storing data in XML format in contrast to a flat, table-like format such as CSV:

\begin{itemize}
\tightlist
\item
  Storing the actual data and metadata in the same file is straightforward (as the above example illustrates). Storing metadata in the first lines of a CSV file (such as in the example above) is theoretically possible. However, by doing so, we break the CSV syntax, and a CSV-parser would likely break down when reading such a file (recall the simple CSV parsing algorithm). More generally, we can represent much more \emph{complex (multi-dimensional)} data in XML files than what is possible in CSVs. In fact, the nesting structure can be arbitrarily complex as long as the XML syntax is valid.
\item
  The XML syntax is largely self-explanatory and thus both \emph{machine-readable and human-readable}. That is, not only can parsers/computers more easily handle complex data structures, but human readers can intuitively understand what the data is all about by looking at the raw XML file.
\end{itemize}

A potential drawback of storing data in XML format is that variable names (in tags) are repeated. Since each tag consists of a couple of bytes, this can be highly inefficient compared to a table-like format where variable names are only defined once. Typically, this means that if the data at hand is only two-dimensional (observations/variables), a CSV format makes more sense.

\hypertarget{deciphering-json}{%
\section{Deciphering JSON}\label{deciphering-json}}

In many web applications, JSON serves the same purpose as XML\footnote{Not that programs running on the server side are frequently capable of returning the same data in either format}. An obvious difference between the two conventions is that JSON does not use tags but attribute-value pairs to annotate data. The following code example shows how the same data can be represented in XML or in JSON (example code taken from \url{https://en.wikipedia.org/wiki/JSON}):

\emph{XML:}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{\textless{}}\KeywordTok{person}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}}\KeywordTok{firstName}\NormalTok{\textgreater{}John\textless{}/}\KeywordTok{firstName}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}}\KeywordTok{lastName}\NormalTok{\textgreater{}Smith\textless{}/}\KeywordTok{lastName}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}}\KeywordTok{age}\NormalTok{\textgreater{}25\textless{}/}\KeywordTok{age}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}}\KeywordTok{address}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{streetAddress}\NormalTok{\textgreater{}21 2nd Street\textless{}/}\KeywordTok{streetAddress}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{city}\NormalTok{\textgreater{}New York\textless{}/}\KeywordTok{city}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{state}\NormalTok{\textgreater{}NY\textless{}/}\KeywordTok{state}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{postalCode}\NormalTok{\textgreater{}10021\textless{}/}\KeywordTok{postalCode}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}/}\KeywordTok{address}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}}\KeywordTok{phoneNumber}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{type}\NormalTok{\textgreater{}home\textless{}/}\KeywordTok{type}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{number}\NormalTok{\textgreater{}212 555{-}1234\textless{}/}\KeywordTok{number}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}/}\KeywordTok{phoneNumber}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}}\KeywordTok{phoneNumber}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{type}\NormalTok{\textgreater{}fax\textless{}/}\KeywordTok{type}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{number}\NormalTok{\textgreater{}646 555{-}4567\textless{}/}\KeywordTok{number}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}/}\KeywordTok{phoneNumber}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}}\KeywordTok{gender}\NormalTok{\textgreater{}}
\NormalTok{    \textless{}}\KeywordTok{type}\NormalTok{\textgreater{}male\textless{}/}\KeywordTok{type}\NormalTok{\textgreater{}}
\NormalTok{  \textless{}/}\KeywordTok{gender}\NormalTok{\textgreater{}}
\NormalTok{\textless{}/}\KeywordTok{person}\NormalTok{\textgreater{}}
\end{Highlighting}
\end{Shaded}

\emph{JSON:}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{\{}\DataTypeTok{"firstName"}\FunctionTok{:} \StringTok{"John"}\FunctionTok{,}
  \DataTypeTok{"lastName"}\FunctionTok{:} \StringTok{"Smith"}\FunctionTok{,}
  \DataTypeTok{"age"}\FunctionTok{:} \DecValTok{25}\FunctionTok{,}
  \DataTypeTok{"address"}\FunctionTok{:} \FunctionTok{\{}
    \DataTypeTok{"streetAddress"}\FunctionTok{:} \StringTok{"21 2nd Street"}\FunctionTok{,}
    \DataTypeTok{"city"}\FunctionTok{:} \StringTok{"New York"}\FunctionTok{,}
    \DataTypeTok{"state"}\FunctionTok{:} \StringTok{"NY"}\FunctionTok{,}
    \DataTypeTok{"postalCode"}\FunctionTok{:} \StringTok{"10021"}
  \FunctionTok{\},}
  \DataTypeTok{"phoneNumber"}\FunctionTok{:} \OtherTok{[}
    \FunctionTok{\{}
      \DataTypeTok{"type"}\FunctionTok{:} \StringTok{"home"}\FunctionTok{,}
      \DataTypeTok{"number"}\FunctionTok{:} \StringTok{"212 555{-}1234"}
    \FunctionTok{\}}\OtherTok{,}
    \FunctionTok{\{}
      \DataTypeTok{"type"}\FunctionTok{:} \StringTok{"fax"}\FunctionTok{,}
      \DataTypeTok{"number"}\FunctionTok{:} \StringTok{"646 555{-}4567"}
    \FunctionTok{\}}
  \OtherTok{]}\FunctionTok{,}
  \DataTypeTok{"gender"}\FunctionTok{:} \FunctionTok{\{}
    \DataTypeTok{"type"}\FunctionTok{:} \StringTok{"male"}
  \FunctionTok{\}}
\FunctionTok{\}}
\end{Highlighting}
\end{Shaded}

Note that despite the syntax differences, the similarities regarding the nesting structure are visible in both formats. For example, \texttt{postalCode} is embedded in \texttt{address}, \texttt{firstname} and \texttt{lastname} are at the same nesting level, etc. The following figure, illustrating the nesting structure further illustrates this point. The basic logic of what element belongs to which higher-level element displayed in the tree diagramm is encoded in both the JSON and the XML excerpts shown above.

\begin{figure}

{\centering \includegraphics[width=0.75\linewidth]{img/hierarch_data} 

}

\caption{XML tree diagram.}\label{fig:xmltree}
\end{figure}



Both XML and JSON are predominantly used to store rather complex, multi-dimensional data. Because they are both human- and machine-readable, these formats are often used to transfer data between applications and users on the Web. Therefore, we encounter these data formats particularly often when we collect data from online sources. Moreover, in the economics/social science context, large and complex data sets (`Big Data') often come along in these formats exactly because of the increasing importance of the Internet as a data source for empirical research in these disciplines.

\hypertarget{parsing-xml-and-json-in-r}{%
\section{Parsing XML and JSON in R}\label{parsing-xml-and-json-in-r}}

As in the case of CSV-like formats, there are several parsers in R that we can use to read XML and JSON data. The following examples are based on the example code shown above (the two text-files \texttt{persons.json} and \texttt{persons.xml})

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(xml2)}

\CommentTok{\# parse XML, represent XML document as R object}
\NormalTok{xml\_doc }\OtherTok{\textless{}{-}} \FunctionTok{read\_xml}\NormalTok{(}\StringTok{"persons.xml"}\NormalTok{)}
\NormalTok{xml\_doc}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## {xml_document}
## <person>
## [1] <firstName>John</firstName>
## [2] <lastName>Smith</lastName>
## [3] <age>25</age>
## [4] <address>\n  <streetAddress>21 2nd Street</streetAddress>\ ...
## [5] <phoneNumber>\n  <type>home</type>\n  <number>212 555-1234 ...
## [6] <phoneNumber>\n  <type>fax</type>\n  <number>646 555-4567< ...
## [7] <gender>\n  <type>male</type>\n</gender>
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(jsonlite)}

\CommentTok{\# parse the JSON{-}document shown in the example above}
\NormalTok{json\_doc }\OtherTok{\textless{}{-}}\NormalTok{ from }\FunctionTok{JSON}\NormalTok{(}\StringTok{"persons.json"}\NormalTok{)}

\CommentTok{\# check the structure}
\FunctionTok{str}\NormalTok{(json\_doc)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## List of 6
##  $ firstName  : chr "John"
##  $ lastName   : chr "Smith"
##  $ age        : int 25
##  $ address    :List of 4
##   ..$ streetAddress: chr "21 2nd Street"
##   ..$ city         : chr "New York"
##   ..$ state        : chr "NY"
##   ..$ postalCode   : chr "10021"
##  $ phoneNumber:'data.frame': 2 obs. of  2 variables:
##   ..$ type  : chr [1:2] "home" "fax"
##   ..$ number: chr [1:2] "212 555-1234" "646 555-4567"
##  $ gender     :List of 1
##   ..$ type: chr "male"
\end{verbatim}

\hypertarget{html}{%
\section{HTML}\label{html}}

Recall the data processing example in which we investigated how a webpage can be downloaded, processed, and stored via the R command line. The code constituting a webpage is written in \href{https://en.wikipedia.org/wiki/HTML}{HyperText Markup Language (HTML)}, designed to be read in a web browser. Thus, HTML is predominantly designed for visual display in browsers, not for storing data. HTML is used to annotate content and define the hierarchy of content in a document to tell the browser how to display (`render') this document on the computer screen. Interestingly for us, its structure is very close to that of XML. However, the tags that can be used are strictly pre-defined and are aimed at explaining the structure of a website.

While not intended as a format to store and transfer data, HTMLdocuments (webpages) have de facto become a very important data source for many data science applications both in industry and academia.\footnote{The systematic collection and extraction of data from such web sources (often referred to as Web Data Mining) goes well beyond the scope of this book.} Note how the two key concepts of computer code and digital data (both residing in a text file) are combined in an HTML document. From a web designer's perspective, HTML is a tool to design the layout of a webpage (and the resulting HTML document is rather seen as \emph{code}). On the other hand, from a data scientist's perspective, HTML gives the data contained in a webpage (the actual content) a certain degree of structure which can be exploited to systematically extract the data from the webpage. In the context of HTML documents/webpages as data sources, we thus also speak of `semi-structured data': a webpage can contain an HTML table (structured data) but likely also contains just raw text (unstructured data). In the following, we explore the basic syntax of HTML by building a simple webpage.

\hypertarget{write-a-simple-webpage-with-html}{%
\subsection{Write a simple webpage with HTML}\label{write-a-simple-webpage-with-html}}

We start by opening a new text-file in RStudio (File-\textgreater New File-\textgreater TextFile). On the first line, we tell the browser what kind of document this is with the \texttt{\textless{}!DOCTYPE\textgreater{}} declaration set to \texttt{HTML}. In addition, the content of the whole HTML document must be put within \texttt{\textless{}html\textgreater{}} and \texttt{\textless{}/html\textgreater{}}, which represents the `root' of the HTML document. In this, you already recognize the typical (XML-like) annotation style in HTML with so-called HTML tags, starting with \texttt{\textless{}} and ending with \texttt{\textgreater{}} or \texttt{/\textgreater{}} in the case of the closing tag, respectively. What is defined between two tags is either another HTML tag or the actual content. An HTML document usually consists of two main components: the head (everything between \texttt{\textless{}head\textgreater{}} and \texttt{\textless{}/head\textgreater{}} ) and the body. The head typically contains metadata describing the whole document like, the title of the document: \texttt{\textless{}title\textgreater{}hello,\ world\textless{}/title\textgreater{}}. The body (everything between \texttt{\textless{}body\textgreater{}} and \texttt{\textless{}/body\textgreater{}}) contains all kinds of specific content: text, images, tables, links, etc. In our very simple example, we add a few words of plain text. We can now save this text document as \texttt{mysite.html} and open it in a web browser.

\begin{Shaded}
\begin{Highlighting}[]
     \DataTypeTok{\textless{}!DOCTYPE }\NormalTok{html}\DataTypeTok{\textgreater{}}

     \KeywordTok{\textless{}html\textgreater{}}
         \KeywordTok{\textless{}head\textgreater{}}
             \KeywordTok{\textless{}title\textgreater{}}\NormalTok{hello, world}\KeywordTok{\textless{}/title\textgreater{}}
         \KeywordTok{\textless{}/head\textgreater{}}
         \KeywordTok{\textless{}body\textgreater{}}
             \KeywordTok{\textless{}h2\textgreater{}}\NormalTok{ hello, world }\KeywordTok{\textless{}/h2\textgreater{}}
         \KeywordTok{\textless{}/body\textgreater{}}
     \KeywordTok{\textless{}/html\textgreater{}}
\end{Highlighting}
\end{Shaded}

From this example, we can learn a few important characteristics of HTML:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  It becomes apparent how HTML is used to annotate/`mark up' data/text (with tags) to define the document's content, structure, and hierarchy. Thus if we want to know what the title of this document is, we have to look for the \texttt{\textless{}title\textgreater{}}-tag.
\item
  This systematic structuring adheres to the nesting principle: `head' and `body' are nested within the `HTML' document, at the same hierarchy level. The `title' is defined within the `head', one level lower. In other words, the `title' is a component of the `head,' which is a component of the `html' document. This logic of encapsulating one part in another holds true in all correctly defined HTML documents. This is the type of `data structure' mentioned above, which can be used to extract specific parts of data from HTML documents in a systematic manner.
\item
  We recognize that HTML code essentially expresses what is what in a document (note the similarity of the concept to storing data in XML). HTML code does not contain explicit instructions like programming languages, telling the computer what to do. If an HTML document contains a link to another website, all that is needed, is to define (with HTML tag \texttt{\textless{}a\ href=...\textgreater{}}) that this is a link. We do not have to explicitly express something like ``if the user clicks on a link, then execute this and that\ldots{}''.
\end{enumerate}

What to do with the HTML document is thus in the hands of the person who works with it. From the data scientist's perspective, we need to know how to traverse and exploit the structure of an HTML document in order to systematically extract the specific content/data that we are interested in. We thus have to learn how to tell the computer (with R) to extract a specific part of an HTML document. And to do so, we have to acquire a basic understanding of the nesting structure implied by HTML (essentially the same logic as with XML).

One way to think about an HTML document is to imagine it as a tree-diagram, with \texttt{\textless{}html\textgreater{}..\textless{}/html\textgreater{}} as the `root', \texttt{\textless{}head\textgreater{}...\textless{}/head\textgreater{}} and \texttt{\textless{}body\textgreater{}...\textless{}/body\textgreater{}} as the `children' of \texttt{\textless{}html\textgreater{}..\textless{}/html\textgreater{}} (and `siblings' of each other), \texttt{\textless{}title\textgreater{}...\textless{}/title\textgreater{}} as the child of \texttt{\textless{}head\textgreater{}...\textless{}/head\textgreater{}}, etc. Figure \ref{fig:html} illustrates this point.

\begin{figure}

{\centering \includegraphics[width=0.5\linewidth]{img/dom_tree} 

}

\caption{HTML (DOM) tree diagram.}\label{fig:html}
\end{figure}



The illustration of the nested structure can help to understand how we can instruct the computer to find/extract a specific part of the data from such a document. In the following exercise, we revisit the example shown in the chapter on data processing to do exactly this.

\hypertarget{two-ways-to-read-a-webpage-into-r}{%
\subsection{Two ways to read a webpage into R}\label{two-ways-to-read-a-webpage-into-r}}

In this example, we look at \href{https://en.wikipedia.org/wiki/Economy_of_Switzerland}{Wikipedia's Economy of Switzerland page}, which contains the table depicted in Figure \ref{fig:swiss}.

\begin{figure}

{\centering \includegraphics[width=0.5\linewidth]{img/1_SwissGDP} 

}

\caption{Source: \url{https://en.wikipedia.org/wiki/Economy_of_Switzerland}.}\label{fig:swiss}
\end{figure}



As in the example shown in the data processing chapter (`world population clock'), we first tell R to read the lines of text from the HTML document that constitutes the webpage.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{swiss\_econ }\OtherTok{\textless{}{-}} \FunctionTok{readLines}\NormalTok{(}\StringTok{"https://en.wikipedia.org/wiki/Economy\_of\_Switzerland"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in readLines("https://en.wikipedia.org/wiki/
## Economy_of_Switzerland"): incomplete final line found on
## 'https://en.wikipedia.org/wiki/Economy_of_Switzerland'
\end{verbatim}

And we can check if everything worked out well by having a look at the first lines of the web page's HTML code:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{head}\NormalTok{(swiss\_econ)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "<!DOCTYPE html>"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
## [2] "<html class=\"client-nojs\" lang=\"en\" dir=\"ltr\">"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   
## [3] "<head>"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 
## [4] "<meta charset=\"UTF-8\"/>"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              
## [5] "<title>Economy of Switzerland - Wikipedia</title>"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      
## [6] "<script>document.documentElement.className=\"client-js\";RLCONF={\"wgBreakFrames\":false,\"wgSeparatorTransformTable\":[\"\",\"\"],\"wgDigitTransformTable\":[\"\",\"\"],\"wgDefaultDateFormat\":\"dmy\",\"wgMonthNames\":[\"\",\"January\",\"February\",\"March\",\"April\",\"May\",\"June\",\"July\",\"August\",\"September\",\"October\",\"November\",\"December\"],\"wgRequestId\":\"15e073fa-3486-4c22-a5f8-689e1bbe3470\",\"wgCSPNonce\":false,\"wgCanonicalNamespace\":\"\",\"wgCanonicalSpecialPageName\":false,\"wgNamespaceNumber\":0,\"wgPageName\":\"Economy_of_Switzerland\",\"wgTitle\":\"Economy of Switzerland\",\"wgCurRevisionId\":1115528632,\"wgRevisionId\":1115528632,\"wgArticleId\":27465,\"wgIsArticle\":true,\"wgIsRedirect\":false,\"wgAction\":\"view\",\"wgUserName\":null,\"wgUserGroups\":[\"*\"],\"wgCategories\":[\"CS1 maint: archived copy as title\",\"CS1 German-language sources (de)\",\"Articles with German-language sources (de)\",\"Webarchive template wayback links\",\"Articles with French-language sources (fr)\",\"Articles with short description\",\"Short description is different from Wikidata\","
\end{verbatim}

The next thing we do is to look at how we can filter the webpage for certain information. For example, we search in the source code for the line that contains the part of the webpage with the table showing data on the Swiss GDP:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{line\_number }\OtherTok{\textless{}{-}} \FunctionTok{grep}\NormalTok{(}\StringTok{\textquotesingle{}US Dollar Exchange\textquotesingle{}}\NormalTok{, swiss\_econ)}
\end{Highlighting}
\end{Shaded}

Recall: we ask R on which line in \texttt{swiss\_econ} (the source code stored in RAM) the text \texttt{US\ Dollar\ Exchange} is and store the answer (the line number) in RAM under the variable name \texttt{line\_number}.

Then, we check on which line the text was found.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{line\_number}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 231
\end{verbatim}

Knowing that the R object \texttt{swiss\_econ} is a character vector (with each element containing one line of HTML code as a character string), we can look at this particular code:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{swiss\_econ[line\_number]}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "<th>US Dollar Exchange"
\end{verbatim}

Note that this rather primitive approach is ok to extract a chunk of code from an HTML document, but it is far from practical when we want to extract specific parts of data (the actual content, not including HTML code). So far, we have completely ignored that the HTML tags give some structure to the data in this document. That is, we simply have read the entire document line by line, not making a difference between code and data. The approach to filter the document could have equally well been taken for a plain text file.

If we want to exploit the structure given by HTML, we need to \emph{parse} the HTML when reading the webpage into R. We can do this with the help of functions provided in the \texttt{rvest} package:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# install package if not yet installed}
\CommentTok{\# install.packages("rvest")}

\CommentTok{\# load the package}
\FunctionTok{library}\NormalTok{(rvest)}
\end{Highlighting}
\end{Shaded}

After loading the package, we read the webpage into R, but this time using a function that parses the HTML code:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# parse the webpage, show the content}
\NormalTok{swiss\_econ\_parsed }\OtherTok{\textless{}{-}} \FunctionTok{read\_html}\NormalTok{(}\StringTok{"https://en.wikipedia.org/wiki/Economy\_of\_Switzerland"}\NormalTok{)}
\NormalTok{swiss\_econ\_parsed}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## {html_document}
## <html class="client-nojs" lang="en" dir="ltr">
## [1] <head>\n<meta http-equiv="Content-Type" content="text/html ...
## [2] <body class="mediawiki ltr sitedir-ltr mw-hide-empty-elt n ...
\end{verbatim}

Now we can easily separate the data/text from the HTML code. For example, we can extract the HTML table containing the data we are interested in as \texttt{data\ frames}.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{tab\_node }\OtherTok{\textless{}{-}} \FunctionTok{html\_node}\NormalTok{(swiss\_econ\_parsed, }\AttributeTok{xpath =} \StringTok{"//*[@id=\textquotesingle{}mw{-}content{-}text\textquotesingle{}]/div/table[2]"}\NormalTok{)}
\NormalTok{tab }\OtherTok{\textless{}{-}} \FunctionTok{html\_table}\NormalTok{(tab\_node)}
\NormalTok{tab}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 19 x 3
##     Year `GDP (billions of CHF)` `US Dollar Exchange`
##    <int>                   <int> <chr>               
##  1  1980                     184 1.67 Francs         
##  2  1985                     244 2.43 Francs         
##  3  1990                     331 1.38 Francs         
##  4  1995                     374 1.18 Francs         
##  5  2000                     422 1.68 Francs         
##  6  2005                     464 1.24 Francs         
##  7  2006                     491 1.25 Francs         
##  8  2007                     521 1.20 Francs         
##  9  2008                     547 1.08 Francs         
## 10  2009                     535 1.09 Francs         
## 11  2010                     546 1.04 Francs         
## 12  2011                     659 0.89 Francs         
## 13  2012                     632 0.94 Francs         
## 14  2013                     635 0.93 Francs         
## 15  2014                     644 0.92 Francs         
## 16  2015                     646 0.96 Francs         
## 17  2016                     659 0.98 Francs         
## 18  2017                     668 1.01 Francs         
## 19  2018                     694 1.00 Francs
\end{verbatim}

\hypertarget{text-data}{%
\chapter{Text Data}\label{text-data}}

\emph{This chapter has been contributed by Aurélien Salling (\href{https://github.com/ASallin}{ASallin})}.

\hypertarget{introduction-1}{%
\section{Introduction}\label{introduction-1}}

Text as data has become increasingly available in the past years, especially following the spread of the Internet and the numerisation of text sources. We are now able to make use of literary texts, analyses (such as financial analyses), reactions on social media (e.g., sentiment analysis to predict financial prices), political discourses, judicial decisions, central bank minutes, etc. in quantitative analyses.

The main challenge of working with text is that text is \textbf{unstructured}. Structured data are coded in such a way that variables can be directly used for analysis without too much loss of content. For instance, data are coded into binary or categorical variables in order to reduce and summarize the complexity of the information. In contrast, with unstructured text data, information is not easily summarized and requires a substantial amount of preparation in order to be used for analysis.

Working with text data can be broken down into eight steps, as depicted in the figure below. (1) Data acquisition: text data are collected from disparate sources. Examples are webpage scrapping, numerization of old administrative records, collection of tweets through an API, etc. (2) Text cleaning and (3) text preprocessing require the analyst to prepare the text in such a way that text information can be read by a statistical software. Most importantly, cleaning and preprocessing are important to remove all the ``noise'' from the text and get the relevant information. For instance, in many applications, punctuation and so-called ``stopwords'' are removed. At this step, text information is ``transformed'' into a matrix. (4) Feature engineering is the process of using domain knowledge to extract features (characteristics, properties, attributes) from raw data. The extraction of the relevant and meaningful features from the text must make sense and be informative in light of the context.

In this introduction, we will focus on steps (1) to (4). Steps (5) to (8) are beyond the scope of a data handling course, as they involve modeling the text through machine learning or other statistical methods, deploying these models for statistical analyses (for research or industrial purpose), and the plausibilization of such models.

\begin{figure}

{\centering \includegraphics[width=0.95\linewidth]{img/nlp_pipeline} 

}

\caption{Illustration of a NLP pipeline.}\label{fig:nlppipeline}
\end{figure}



\hypertarget{one-package-to-rule-them-all-quanteda}{%
\section{One Package To Rule Them All: Quanteda}\label{one-package-to-rule-them-all-quanteda}}

In R, steps (2) to (4) can be performed using the \texttt{quanteda} or the \texttt{tidytext} package. \texttt{tidytext} works very well with the packages of the \texttt{tidyverse} family, such as \texttt{dplyr} or \texttt{tidyr}. This package converts text to and from tidy formats.

The package \texttt{quanteda} remains, however, the most complete and go-to package for text analysis in R. It allows R users to perform all the preprocessing steps very easily, and provides all the functions necessary for preprocessing, visualization, and even statistical analyses of the corpus.

\hfill\break

\hypertarget{from-raw-text-to-corpus-step-1}{%
\subsection{From raw text to corpus: step (1)}\label{from-raw-text-to-corpus-step-1}}

The base, raw material, of quantitative text analysis is a \textbf{corpus}. A corpus is, in NLP, \emph{a collection of authentic text organized into datasets}. A corpus can be made up of everything from newspapers, novels, recipes, radio broadcasts to television shows, movies, and tweets. Typically, the data structure of a corpus organizes the individual text documents (e.g., tweets) as individual components of a larger object (the entire corpus), whereby each document-component is linked to its corresponding metadata-component (containing data describing the characteristics of a document; e.g., the timestamp a tweet was posted, the author of the tweet, etc.). The following figure illustrates this data structure for a generic text corpus.

\begin{figure}

{\centering \includegraphics[width=0.25\linewidth]{img/text_corpus} 

}

\caption{Illustration of the data structure behind a text corpus.}\label{fig:corpus}
\end{figure}



In the specific case of \texttt{quanteda}, a corpus is a \textbf{a data frame consisting of a character vector for documents, and additional vectors for document-level variables}. In other words, a corpus is a data frame that contains, in each row, a text document, and additional columns with the corresponding metadata about the text.

Text in a raw form is often found in a \texttt{.json} format (after web scraping), in a \texttt{.csv} format, or in simple \texttt{.txt} files. The first task is then to import the text data in R and transform it as a corpus. For this introduction, we will use the \texttt{inauguration} corpus from \texttt{quanteda}, which is a standard corpus used in introductory text analysis. It contains the inauguration discourses of the five first US presidents. This text data can be loaded from the \texttt{readtext} package. The text is contained in a csv file, and is loaded with the \texttt{read.csv()} function. The metadata of this corpus is the year of the inauguration and the name of the president taking office.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# set path}
\NormalTok{path\_data }\OtherTok{\textless{}{-}} \FunctionTok{system.file}\NormalTok{(}\StringTok{"extdata/"}\NormalTok{, }\AttributeTok{package =} \StringTok{"readtext"}\NormalTok{)}

\CommentTok{\# import csv file}
\NormalTok{dat\_inaug }\OtherTok{\textless{}{-}} \FunctionTok{read.csv}\NormalTok{(}\FunctionTok{paste0}\NormalTok{(path\_data, }\StringTok{"/csv/inaugCorpus.csv"}\NormalTok{))}
\FunctionTok{names}\NormalTok{(dat\_inaug)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "texts"     "Year"      "President" "FirstName"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Create a corpus}
\NormalTok{corp }\OtherTok{\textless{}{-}} \FunctionTok{corpus}\NormalTok{(dat\_inaug, }\AttributeTok{text\_field =} \StringTok{"texts"}\NormalTok{)}
\FunctionTok{print}\NormalTok{(corp)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Corpus consisting of 5 documents and 3 docvars.
## text1 :
## "Fellow-Citizens of the Senate and of the House of Representa..."
## 
## text2 :
## "Fellow citizens, I am again called upon by the voice of my c..."
## 
## text3 :
## "When it was first perceived, in early times, that no middle ..."
## 
## text4 :
## "Friends and Fellow Citizens: Called upon to undertake the du..."
## 
## text5 :
## "Proceeding, fellow citizens, to that qualification which the..."
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Look at the metadata in the corpus using \textasciigrave{}docvars\textasciigrave{}}
\FunctionTok{docvars}\NormalTok{(corp)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   Year  President FirstName
## 1 1789 Washington    George
## 2 1793 Washington    George
## 3 1797      Adams      John
## 4 1801  Jefferson    Thomas
## 5 1805  Jefferson    Thomas
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# In quanteda, the metadata in a corpus can be handled like data frames.}
\FunctionTok{docvars}\NormalTok{(corp, }\AttributeTok{field =} \StringTok{"Century"}\NormalTok{) }\OtherTok{\textless{}{-}} \FunctionTok{floor}\NormalTok{(}\FunctionTok{docvars}\NormalTok{(corp, }\AttributeTok{field =} \StringTok{"Year"}\NormalTok{) }\SpecialCharTok{/} \DecValTok{100}\NormalTok{) }\SpecialCharTok{+} \DecValTok{1}
\end{Highlighting}
\end{Shaded}

\hfill\break

\hypertarget{text-the-traditional-way-regular-expressions}{%
\subsubsection{Text the traditional way: regular expressions}\label{text-the-traditional-way-regular-expressions}}

A first basic step to extract simple information from written information is to use traditional \textbf{regular expressions}. Regular expressions are a flexible syntax used to detect and describe patterns in strings. They are used not only in R, but in other softwares. For an introduction, see \url{https://rstudio-pubs-static.s3.amazonaws.com/74603_76cd14d5983f47408fdf0b323550b846.html}. Regular expressions in R can be used with \texttt{base} R, but also with more practical packages such as \texttt{stringr} and \texttt{stringi}.

In the following example, we use regular expressions to count the number of occurences of the word \emph{peace} in the inaugural discourses using the command \texttt{str\_count()} from \texttt{stringr}. In an additional step, we count the number of occurences of the pronoun ``I''. Finally, we program a regular expression to extract the first five words of each discourse.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Count occurences of the word "peace"}
\FunctionTok{str\_count}\NormalTok{(corp, }\StringTok{"[Pp]eace"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 0 0 5 7 4
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Count occurences of the words "peace" OR "war"}
\FunctionTok{str\_count}\NormalTok{(corp, }\StringTok{"[Pp]eace|[Ww]ar"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1]  1  0 10 10  8
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Count occurences of the mention of the first person pronoun "I"}
\FunctionTok{str\_count}\NormalTok{(corp, }\StringTok{"I"}\NormalTok{) }\CommentTok{\# counts the number of "I" occurences. This is not what we want.}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 30  6 24 23 28
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{str\_count}\NormalTok{(corp, }\StringTok{"[I][[:space:]]"}\NormalTok{) }\CommentTok{\# counts the number of "I" followed by a space.}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 23  6 13 21 18
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Extract the first five words of each discourse}
\FunctionTok{str\_extract}\NormalTok{(corp, }\StringTok{"\^{}(}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{S+}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{s|[[:punct:]]|}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{n)\{5\}"}\NormalTok{) }\CommentTok{\# \^{}serves to anchor at the beginning of the string, ()\{5\} shows the group of signs must be detected five times. \textbackslash{}S if for any non{-}space character,  \textbackslash{}s is for space, [[:punct:]] for punctuation, and \textbackslash{}n for the string representation of paragraphs. Basically, it means: five the first five occurences of many non{-}space characters (+) followed either (|) by a space, a punctuation sign, or a paragraph sign.}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "Fellow-Citizens of the Senate and "   
## [2] "Fellow citizens, I am again "         
## [3] "When it was first perceived, "        
## [4] "Friends and Fellow Citizens:\n\n"     
## [5] "Proceeding, fellow citizens, to that "
\end{verbatim}

\hfill\break

\hypertarget{from-corpus-to-tokens-steps-2-and-3}{%
\subsection{From corpus to tokens: steps (2) and (3)}\label{from-corpus-to-tokens-steps-2-and-3}}

Once we have a corpus, we want to extract the substance of the text. This means, in \texttt{quanteda} language, that we want to extract \textbf{tokens}, i.e.~to isolate the elements that constitute a corpus in order to, in a second phase, to quantify them. Basically, tokens are expressions that form the building blocks (both grammatically and from a meaning perspective) of the text. In many quantitative text analyses, researchers want to remove all words that are unlikely to be informative of a document's content. This means that punctuation as well as so-called ``stopwords'' (e.g.~``and'', ``a'', ``the'', ``or'', ``but'', etc.) are removed.

In our example, we first create tokens from each document of our corpus. For instance, the first document of the corpus is made out of 1537 tokens. Most of these tokens are not relevant. So we proceed by removing punctuation and stopwords. \texttt{quanteda} has a built-in list of stopwords in English. We also want to remove the words \emph{fellow} and \emph{citizens}, as those words appear in each document.

Finally, not only are tokens for single words interesting, but also tokens of combined words (called ``N-grams''). N-grams are a sequence of tokens from already tokenized text objects. Usually, these N-grams are generated in all possible combinations of tokens. For instance, if the expression ``not friendly'' appears often, we might lose valuable information when building tokens that do not account for the co-occurence of these terms. We identify bigrams with the word ``never''.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{toks }\OtherTok{\textless{}{-}} \FunctionTok{tokens}\NormalTok{(corp)}
\FunctionTok{head}\NormalTok{(toks[[}\DecValTok{1}\NormalTok{]], }\DecValTok{20}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##  [1] "Fellow-Citizens" "of"              "the"            
##  [4] "Senate"          "and"             "of"             
##  [7] "the"             "House"           "of"             
## [10] "Representatives" ":"               "Among"          
## [13] "the"             "vicissitudes"    "incident"       
## [16] "to"              "life"            "no"             
## [19] "event"           "could"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Remove punctuation}
\NormalTok{toks }\OtherTok{\textless{}{-}} \FunctionTok{tokens}\NormalTok{(corp, }\AttributeTok{remove\_punct =} \ConstantTok{TRUE}\NormalTok{)}
\FunctionTok{head}\NormalTok{(toks[[}\DecValTok{1}\NormalTok{]], }\DecValTok{20}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##  [1] "Fellow-Citizens" "of"              "the"            
##  [4] "Senate"          "and"             "of"             
##  [7] "the"             "House"           "of"             
## [10] "Representatives" "Among"           "the"            
## [13] "vicissitudes"    "incident"        "to"             
## [16] "life"            "no"              "event"          
## [19] "could"           "have"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Remove stopwords}
\FunctionTok{stopwords}\NormalTok{(}\StringTok{"en"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   [1] "i"          "me"         "my"         "myself"    
##   [5] "we"         "our"        "ours"       "ourselves" 
##   [9] "you"        "your"       "yours"      "yourself"  
##  [13] "yourselves" "he"         "him"        "his"       
##  [17] "himself"    "she"        "her"        "hers"      
##  [21] "herself"    "it"         "its"        "itself"    
##  [25] "they"       "them"       "their"      "theirs"    
##  [29] "themselves" "what"       "which"      "who"       
##  [33] "whom"       "this"       "that"       "these"     
##  [37] "those"      "am"         "is"         "are"       
##  [41] "was"        "were"       "be"         "been"      
##  [45] "being"      "have"       "has"        "had"       
##  [49] "having"     "do"         "does"       "did"       
##  [53] "doing"      "would"      "should"     "could"     
##  [57] "ought"      "i'm"        "you're"     "he's"      
##  [61] "she's"      "it's"       "we're"      "they're"   
##  [65] "i've"       "you've"     "we've"      "they've"   
##  [69] "i'd"        "you'd"      "he'd"       "she'd"     
##  [73] "we'd"       "they'd"     "i'll"       "you'll"    
##  [77] "he'll"      "she'll"     "we'll"      "they'll"   
##  [81] "isn't"      "aren't"     "wasn't"     "weren't"   
##  [85] "hasn't"     "haven't"    "hadn't"     "doesn't"   
##  [89] "don't"      "didn't"     "won't"      "wouldn't"  
##  [93] "shan't"     "shouldn't"  "can't"      "cannot"    
##  [97] "couldn't"   "mustn't"    "let's"      "that's"    
## [101] "who's"      "what's"     "here's"     "there's"   
## [105] "when's"     "where's"    "why's"      "how's"     
## [109] "a"          "an"         "the"        "and"       
## [113] "but"        "if"         "or"         "because"   
## [117] "as"         "until"      "while"      "of"        
## [121] "at"         "by"         "for"        "with"      
## [125] "about"      "against"    "between"    "into"      
## [129] "through"    "during"     "before"     "after"     
## [133] "above"      "below"      "to"         "from"      
## [137] "up"         "down"       "in"         "out"       
## [141] "on"         "off"        "over"       "under"     
## [145] "again"      "further"    "then"       "once"      
## [149] "here"       "there"      "when"       "where"     
## [153] "why"        "how"        "all"        "any"       
## [157] "both"       "each"       "few"        "more"      
## [161] "most"       "other"      "some"       "such"      
## [165] "no"         "nor"        "not"        "only"      
## [169] "own"        "same"       "so"         "than"      
## [173] "too"        "very"       "will"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{toks }\OtherTok{\textless{}{-}} \FunctionTok{tokens\_remove}\NormalTok{(toks, }\AttributeTok{pattern =} \FunctionTok{stopwords}\NormalTok{(}\StringTok{"en"}\NormalTok{))}
\FunctionTok{head}\NormalTok{(toks[[}\DecValTok{1}\NormalTok{]], }\DecValTok{20}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##  [1] "Fellow-Citizens" "Senate"          "House"          
##  [4] "Representatives" "Among"           "vicissitudes"   
##  [7] "incident"        "life"            "event"          
## [10] "filled"          "greater"         "anxieties"      
## [13] "notification"    "transmitted"     "order"          
## [16] "received"        "14th"            "day"            
## [19] "present"         "month"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# We can keep words we are interested in}
\FunctionTok{tokens\_select}\NormalTok{(toks, }\AttributeTok{pattern =} \FunctionTok{c}\NormalTok{(}\StringTok{"peace"}\NormalTok{, }\StringTok{"war"}\NormalTok{, }\StringTok{"great*"}\NormalTok{, }\StringTok{"unit*"}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Tokens consisting of 5 documents and 4 docvars.
## text1 :
## [1] "greater" "United"  "Great"   "United"  "united"  "great"  
## [7] "great"   "united" 
## 
## text2 :
## [1] "united"
## 
## text3 :
##  [1] "war"    "great"  "United" "great"  "great"  "peace" 
##  [7] "great"  "peace"  "peace"  "United" "peace"  "peace" 
## [ ... and 2 more ]
## 
## text4 :
##  [1] "greatness" "unite"     "unite"     "greater"   "peace"    
##  [6] "peace"     "peace"     "war"       "peace"     "greatest" 
## [11] "greatest"  "great"    
## [ ... and 1 more ]
## 
## text5 :
## [1] "United" "peace"  "great"  "war"    "war"    "War"   
## [7] "peace"  "peace"  "peace"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Remove "fellow" and "citizen"}
\NormalTok{toks }\OtherTok{\textless{}{-}} \FunctionTok{tokens\_remove}\NormalTok{(toks, }\AttributeTok{pattern =} \FunctionTok{c}\NormalTok{(}
    \StringTok{"fellow*"}\NormalTok{,}
    \StringTok{"citizen*"}\NormalTok{,}
    \StringTok{"senate"}\NormalTok{,}
    \StringTok{"house"}\NormalTok{,}
    \StringTok{"representative*"}\NormalTok{,}
    \StringTok{"constitution"}
\NormalTok{))}

\CommentTok{\# Build N{-}grams (onegrams, bigrams, and 3{-}grams)}
\NormalTok{toks\_ngrams }\OtherTok{\textless{}{-}} \FunctionTok{tokens\_ngrams}\NormalTok{(toks, }\AttributeTok{n =} \DecValTok{2}\SpecialCharTok{:}\DecValTok{3}\NormalTok{)}

\CommentTok{\# Build N{-}grams based on a structure: keep n{-}grams that containt a "not"}
\NormalTok{toks\_neg\_bigram\_select }\OtherTok{\textless{}{-}} \FunctionTok{tokens\_select}\NormalTok{(toks\_ngrams, }\AttributeTok{pattern =} \FunctionTok{phrase}\NormalTok{(}\StringTok{"never\_*"}\NormalTok{))}
\FunctionTok{head}\NormalTok{(toks\_neg\_bigram\_select[[}\DecValTok{1}\NormalTok{]], }\DecValTok{30}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "never_hear"            "never_expected"       
## [3] "never_hear_veneration" "never_expected_nation"
\end{verbatim}

\hfill\break

\hypertarget{from-tokens-to-document-term-matrix-dtm-steps-3-and-4}{%
\subsection{From tokens to document-term-matrix (dtm): steps (3) and (4)}\label{from-tokens-to-document-term-matrix-dtm-steps-3-and-4}}

The last step of our introduction is to make our collection of tokens usable for quantitative analysis, i.e.~to set the tokens into a structure that we can use to train machine learning models, compute statistics, word frequencies, or add to regression models.

Tokens are usually represented as a document-term-matrix (dtm, also known as document-feature-matrix, dfm). dtms have as rows the document, and as columns the tokens. They contain the count frequency, or sometimes an indicator for whether a given token appears in a document. To create a dtm, we can use \texttt{quanteda}'s \texttt{dfm} command, as shown below.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{dfmat }\OtherTok{\textless{}{-}} \FunctionTok{dfm}\NormalTok{(toks)}
\FunctionTok{print}\NormalTok{(dfmat)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Document-feature matrix of: 5 documents, 1,817 features (72.31% sparse) and 4 docvars.
##        features
## docs    among vicissitudes incident life event filled greater
##   text1     1            1        1    1     2      1       1
##   text2     0            0        0    0     0      0       0
##   text3     4            0        0    2     0      0       0
##   text4     1            0        0    1     0      0       1
##   text5     7            0        0    2     0      0       0
##        features
## docs    anxieties notification transmitted
##   text1         1            1           1
##   text2         0            0           0
##   text3         0            0           0
##   text4         0            0           0
##   text5         0            0           0
## [ reached max_nfeat ... 1,807 more features ]
\end{verbatim}

Our dtm has five rows for our five documents, and 6,694 (!) columns. Each column is a single token. The dtm is 79.88\% sparse, which means that 79.88\% of the cells are 0. Handling such large matrices is not trivial, as the number of columns might in many cases increase much faster than the number of rows. For this reason, it is important to clean the text, to remove stopwords, and to be selective about which tokens are kept in the dtm. Moreover, it might sound like a good idea to include N-grams at first, but N-grams are less likely to be matched across documents, which increases the dimension and sparsity of the matrix even more.

The importance of feature engineering and the need to use domain knowledge becomes clear at this point. Everyone working with text faces a tradeoff between quantity of information and quality of information. Moreover, dts that are too large will be difficult to model and used in subsequent analyses. Domain knowledge is necessary to reduce the dimensino of the dtm matrix. Because our dtm is too large and not informative, we want to trim it and remove columns based on their frequencies. When removing tokens that appear less than two times, we are left with a dtm of 72 columns.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{dfmat }\OtherTok{\textless{}{-}} \FunctionTok{dfm}\NormalTok{(toks)}
\NormalTok{dfmat }\OtherTok{\textless{}{-}} \FunctionTok{dfm\_trim}\NormalTok{(dfmat, }\AttributeTok{min\_termfreq =} \DecValTok{2}\NormalTok{) }\CommentTok{\# remove tokens that appear less than 1 times}
\end{Highlighting}
\end{Shaded}

~

\hypertarget{from-dtm-to-analysis-and-insights}{%
\subsection{From dtm to analysis and insights}\label{from-dtm-to-analysis-and-insights}}

Dtms are the basis of all text analyses. From a dtm, we can train machine learning methods to predict the sentiment of a text, to classify the documents into clusters (for instance, classifying spam and non-spam emails), to retrieve missing information, or to predict the autorship.

Very basic statistics about documents are the \textbf{top features} of each document, the frequency of expressions in the corpus

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{topfeatures}\NormalTok{(dfmat, }\DecValTok{10}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## government        may     public        can     people 
##         40         38         30         27         27 
##      shall    country      every         us    nations 
##         23         22         20         20         18
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{tstat\_freq }\OtherTok{\textless{}{-}} \FunctionTok{textstat\_frequency}\NormalTok{(dfmat, }\AttributeTok{n =} \DecValTok{5}\NormalTok{)}

\FunctionTok{textplot\_wordcloud}\NormalTok{(dfmat, }\AttributeTok{max\_words =} \DecValTok{100}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/stats-1.pdf}

\hypertarget{image-data}{%
\chapter{Image data}\label{image-data}}

\emph{This chapter has substantially benefitted from contributions by Michael Tüting (\href{https://github.com/MTueting}{MTueting})}.

With the advent of broadband internet and efforts in digitizing analogue archives a large part of the world's available data is stored in the form of images (and moving images). Subsequent advances in computer vision (machine learning techniques focused on images and videos) have made usage of images for economic and social science purposes accessible to non-computer science researchers. Examples of how image data is used in economic research involve the quantification of urban appearance (based on the analysis of street-level images in cities, \citet{naik_etal2016}), the digitization of old text documents via optical character recognition (\citet{cesarini_etal2016}), and the measurement of economic development/wealth with nighttime light-intensity based on satellite image data (\citet{hodler_raschky2014}). In the following subsections, you will first explore the most common image formats, and how the data behind digital images is structured and stored.

\hypertarget{image-data-structure-and-storage}{%
\section{Image data structure and storage}\label{image-data-structure-and-storage}}

There are two important variants of how digital images are stored: raster images (for example, jpg files), and vector-based images (for example, eps files). In terms of data structure, the two formats differ quite substantially:

\begin{itemize}
\item
  The data structure behind raster images basically defines an image as a matrix of pixels, as well as the color of each pixel. Thereby, screen colors are combinations of the three base colors red, green, blue (RGB). Technically speaking, a raster image thus consists of an array of three matrices with the same dimension (one for each base color). The values of each array element are then indicating the intensity of each base color for the corresponding pixel. For example, a black pixel would be indicated as (0,0,0). Raster images play a major role in essentially all modern computer vision applications related to social science/economic research. Photos, videos, satellite imagery, and scans of old documents all stored as raster images.
\item
  Vector images are essentially text files that store the coordinates of points on a surface and how these dots are connected (or not) by lines. Some of the commonly used formats to store vector images are based on XML (for example SVG files). The most common encounter of vector image files in a data analytics/research context are typically storing map data. Streets, borders, rivers, etc. can be defined as polygons (consisting of individual lines/vectors).
\end{itemize}

Given the fundamental differences in how the data behind these two types of images is structured, practically handling such data in R differs quite substantially between the two formats with regard to the R packages used and the representation of the image object in RAM.

\hypertarget{raster-images-in-r}{%
\section{Raster images in R}\label{raster-images-in-r}}

This is meant to showcase some of the most frequent tasks related to images in R.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Load two common packages}
\FunctionTok{library}\NormalTok{(raster) }
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Loading required package: sp
\end{verbatim}

\hypertarget{basic-data-structure}{%
\subsection{Basic data structure}\label{basic-data-structure}}

Recall that raster images are stored as arrays (\(X\times Y \times Z\)). \(X\) and \(Y\) define the number of pixels in each column and row (width and height) of the image and \(Z\) the number of layers. Greyscale images usually have only one layer, whereas most colored images have 3 layers (such as in the case of RGB images). The following figure illustrates this point based on a \(3 \times 3\) pixels bitmap image.

\begin{figure}

{\centering \includegraphics[width=0.75\linewidth]{img/rgb_structure} 

}

\caption{Illustration of a bitmap file data structure (with RGB schema). Panel A shows a 3x3 pixels bitmap image as it is shown on screen. Panel B illustrates the corresponding bitmap file's data structure: a three-dimensional array with 3x3 elements represents the image matrix in the three base screen colors red, green, and blue. As a reading example, consider the highlighted upper left pixel in A with the correspondingly highlighted values in the RGB array in B. The color of this pixel is RGB code 255 (full red), 51 (some green), and 51 (some blue): (255,51,51).}\label{fig:rgb}
\end{figure}



To get a better feeling for the corresponding data structure in R, we start with generating RGB-images step-by-step in R. First we generate three matrices (one for each base color), arrange these matrices in an array, and then save the plots to disk.

\hypertarget{example-1-generating-a-red-image-rgb-code-25500}{%
\subsubsection{Example 1: Generating a red image (RGB code: 255,0,0)}\label{example-1-generating-a-red-image-rgb-code-25500}}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 1: Define the width and height of the image}
\NormalTok{width }\OtherTok{=} \DecValTok{300}\NormalTok{; }
\NormalTok{height }\OtherTok{=} \DecValTok{300}

\CommentTok{\# Step 2: Define the number of layers (RGB = 3)}
\NormalTok{layers }\OtherTok{=} \DecValTok{3}

\CommentTok{\# Step 3: Generate three matrices corresponding to Red, Green, and Blue values}
\NormalTok{red }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(}\DecValTok{255}\NormalTok{, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}
\NormalTok{green }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(}\DecValTok{0}\NormalTok{, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}
\NormalTok{blue }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(}\DecValTok{0}\NormalTok{, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}

\CommentTok{\# Step 4: Generate an array by combining the three matrices}
\NormalTok{image.array }\OtherTok{=} \FunctionTok{array}\NormalTok{(}\FunctionTok{c}\NormalTok{(red, green, blue), }\AttributeTok{dim =} \FunctionTok{c}\NormalTok{(width, height, layers))}
\FunctionTok{dim}\NormalTok{(image.array)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 300 300   3
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 5: Create RasterBrick}
\NormalTok{image }\OtherTok{=} \FunctionTok{brick}\NormalTok{(image.array)}
\FunctionTok{print}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## class      : RasterBrick 
## dimensions : 300, 300, 90000, 3  (nrow, ncol, ncell, nlayers)
## resolution : 0.003333, 0.003333  (x, y)
## extent     : 0, 1, 0, 1  (xmin, xmax, ymin, ymax)
## crs        : NA 
## source     : memory
## names      : layer.1, layer.2, layer.3 
## min values :     255,       0,       0 
## max values :     255,       0,       0
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 6: Plot RGB}
\FunctionTok{plotRGB}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in .couldBeLonLat(x, warnings = warnings): CRS is NA.
## Assuming it is longitude/latitude
\end{verbatim}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-81-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 7: (Optional) Save to disk}
\FunctionTok{png}\NormalTok{(}\AttributeTok{filename =} \StringTok{"red.png"}\NormalTok{, }\AttributeTok{width =}\NormalTok{ width, }\AttributeTok{height =}\NormalTok{ height, }\AttributeTok{units =} \StringTok{"px"}\NormalTok{)}
\FunctionTok{plotRGB}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in .couldBeLonLat(x, warnings = warnings): CRS is NA.
## Assuming it is longitude/latitude
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{dev.off}\NormalTok{()}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## cairo_pdf 
##         2
\end{verbatim}

\hypertarget{example-2-generating-a-green-image-rgb-code-0-255-0}{%
\subsubsection{Example 2: Generating a green image (RGB code: 0, 255, 0)}\label{example-2-generating-a-green-image-rgb-code-0-255-0}}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 1: Define the width and height of the image}
\NormalTok{width }\OtherTok{=} \DecValTok{300}\NormalTok{; }
\NormalTok{height }\OtherTok{=} \DecValTok{300}

\CommentTok{\# Step 2: Define the number of layers (RGB = 3)}
\NormalTok{layers }\OtherTok{=} \DecValTok{3}

\CommentTok{\# Step 3: Generate three matrices corresponding to Red, Green, and Blue values}
\NormalTok{red }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(}\DecValTok{0}\NormalTok{, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}
\NormalTok{green }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(}\DecValTok{255}\NormalTok{, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}
\NormalTok{blue }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(}\DecValTok{0}\NormalTok{, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}

\CommentTok{\# Step 4: Generate an array by combining the three matrices}
\NormalTok{image.array }\OtherTok{=} \FunctionTok{array}\NormalTok{(}\FunctionTok{c}\NormalTok{(red, green, blue), }\AttributeTok{dim =} \FunctionTok{c}\NormalTok{(width, height, layers))}
\FunctionTok{dim}\NormalTok{(image.array)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 300 300   3
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 5: Create RasterBrick}
\NormalTok{image }\OtherTok{=} \FunctionTok{brick}\NormalTok{(image.array)}
\FunctionTok{print}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## class      : RasterBrick 
## dimensions : 300, 300, 90000, 3  (nrow, ncol, ncell, nlayers)
## resolution : 0.003333, 0.003333  (x, y)
## extent     : 0, 1, 0, 1  (xmin, xmax, ymin, ymax)
## crs        : NA 
## source     : memory
## names      : layer.1, layer.2, layer.3 
## min values :       0,     255,       0 
## max values :       0,     255,       0
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 6: Plot RGB}
\FunctionTok{plotRGB}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in .couldBeLonLat(x, warnings = warnings): CRS is NA.
## Assuming it is longitude/latitude
\end{verbatim}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-82-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 7: (Optional) Save to disk}
\FunctionTok{png}\NormalTok{(}\AttributeTok{filename =} \StringTok{"blue.png"}\NormalTok{, }\AttributeTok{width =}\NormalTok{ width, }\AttributeTok{height =}\NormalTok{ height, }\AttributeTok{units =} \StringTok{"px"}\NormalTok{)}
\FunctionTok{plotRGB}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in .couldBeLonLat(x, warnings = warnings): CRS is NA.
## Assuming it is longitude/latitude
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{dev.off}\NormalTok{()}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## cairo_pdf 
##         2
\end{verbatim}

\hypertarget{example-3-generating-a-blue-image-rgb-code-0-0-255}{%
\subsubsection{Example 3: Generating a blue image (RGB code: 0, 0, 255)}\label{example-3-generating-a-blue-image-rgb-code-0-0-255}}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 1: Define the width and height of the image}
\NormalTok{width }\OtherTok{=} \DecValTok{300}\NormalTok{; }
\NormalTok{height }\OtherTok{=} \DecValTok{300}

\CommentTok{\# Step 2: Define the number of layers (RGB = 3)}
\NormalTok{layers }\OtherTok{=} \DecValTok{3}

\CommentTok{\# Step 3: Generate three matrices corresponding to Red, Green, and Blue values}
\NormalTok{red }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(}\DecValTok{0}\NormalTok{, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}
\NormalTok{green }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(}\DecValTok{0}\NormalTok{, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}
\NormalTok{blue }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(}\DecValTok{255}\NormalTok{, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}

\CommentTok{\# Step 4: Generate an array by combining the three matrices}
\NormalTok{image.array }\OtherTok{=} \FunctionTok{array}\NormalTok{(}\FunctionTok{c}\NormalTok{(red, green, blue), }\AttributeTok{dim =} \FunctionTok{c}\NormalTok{(width, height, layers))}
\FunctionTok{dim}\NormalTok{(image.array)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 300 300   3
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 5: Create RasterBrick}
\NormalTok{image }\OtherTok{=} \FunctionTok{brick}\NormalTok{(image.array)}
\FunctionTok{print}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## class      : RasterBrick 
## dimensions : 300, 300, 90000, 3  (nrow, ncol, ncell, nlayers)
## resolution : 0.003333, 0.003333  (x, y)
## extent     : 0, 1, 0, 1  (xmin, xmax, ymin, ymax)
## crs        : NA 
## source     : memory
## names      : layer.1, layer.2, layer.3 
## min values :       0,       0,     255 
## max values :       0,       0,     255
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 6: Plot RGB}
\FunctionTok{plotRGB}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in .couldBeLonLat(x, warnings = warnings): CRS is NA.
## Assuming it is longitude/latitude
\end{verbatim}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-83-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 7: (Optional) Save to disk}
\FunctionTok{png}\NormalTok{(}\AttributeTok{filename =} \StringTok{"green.png"}\NormalTok{, }\AttributeTok{width =}\NormalTok{ width, }\AttributeTok{height =}\NormalTok{ height, }\AttributeTok{units =} \StringTok{"px"}\NormalTok{)}
\FunctionTok{plotRGB}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in .couldBeLonLat(x, warnings = warnings): CRS is NA.
## Assuming it is longitude/latitude
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{dev.off}\NormalTok{()}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## cairo_pdf 
##         2
\end{verbatim}

\hypertarget{example-4-generating-a-random-rgb-image}{%
\subsubsection{Example 4: Generating a random RGB image}\label{example-4-generating-a-random-rgb-image}}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 1: Define the width and height of the image}
\NormalTok{width }\OtherTok{=} \DecValTok{300}\NormalTok{; }
\NormalTok{height }\OtherTok{=} \DecValTok{300}

\CommentTok{\# Step 2: Define the number of layers (RGB = 3)}
\NormalTok{layers }\OtherTok{=} \DecValTok{3}

\CommentTok{\# Step 3: Draw random color intensities from a standard{-}normal distribution}
\NormalTok{shades\_of\_red }\OtherTok{=} \FunctionTok{rnorm}\NormalTok{(}\AttributeTok{n =}\NormalTok{ width}\SpecialCharTok{*}\NormalTok{height, }\AttributeTok{mean =} \DecValTok{0}\NormalTok{, }\AttributeTok{sd =} \DecValTok{1}\NormalTok{)}
\NormalTok{shades\_of\_green }\OtherTok{=} \FunctionTok{rnorm}\NormalTok{(}\AttributeTok{n =}\NormalTok{ width}\SpecialCharTok{*}\NormalTok{height, }\AttributeTok{mean =} \DecValTok{0}\NormalTok{, }\AttributeTok{sd =} \DecValTok{1}\NormalTok{)}
\NormalTok{shades\_of\_blue }\OtherTok{=} \FunctionTok{rnorm}\NormalTok{(}\AttributeTok{n =}\NormalTok{ width}\SpecialCharTok{*}\NormalTok{height, }\AttributeTok{mean =} \DecValTok{0}\NormalTok{, }\AttributeTok{sd =} \DecValTok{1}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

The color intensity must be in the range 0 to 255, however, our values are
standard normally distributed around 0:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{plot}\NormalTok{(}\FunctionTok{density}\NormalTok{(shades\_of\_red))}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-85-1.pdf}
We first normalize them to a range of 0 to 1 using the formula:
\(z_i = \frac{x_i - min(x)}{max(x)-min(x)}\)
and subsequently multiply by 255:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 4: Normalize to 0,255 range of values}
\NormalTok{shades\_of\_red }\OtherTok{=}\NormalTok{ (shades\_of\_red }\SpecialCharTok{{-}} \FunctionTok{min}\NormalTok{(shades\_of\_red))}\SpecialCharTok{/}\NormalTok{(}\FunctionTok{max}\NormalTok{(shades\_of\_red)}\SpecialCharTok{{-}}\FunctionTok{min}\NormalTok{(shades\_of\_red))}\SpecialCharTok{*}\DecValTok{255}
\NormalTok{shades\_of\_green }\OtherTok{=}\NormalTok{ (shades\_of\_green }\SpecialCharTok{{-}} \FunctionTok{min}\NormalTok{(shades\_of\_green))}\SpecialCharTok{/}\NormalTok{(}\FunctionTok{max}\NormalTok{(shades\_of\_green)}\SpecialCharTok{{-}}\FunctionTok{min}\NormalTok{(shades\_of\_green))}\SpecialCharTok{*}\DecValTok{255}
\NormalTok{shades\_of\_blue }\OtherTok{=}\NormalTok{ (shades\_of\_blue }\SpecialCharTok{{-}} \FunctionTok{min}\NormalTok{(shades\_of\_blue))}\SpecialCharTok{/}\NormalTok{(}\FunctionTok{max}\NormalTok{(shades\_of\_blue)}\SpecialCharTok{{-}}\FunctionTok{min}\NormalTok{(shades\_of\_blue))}\SpecialCharTok{*}\DecValTok{255}

\FunctionTok{plot}\NormalTok{(}\FunctionTok{density}\NormalTok{(shades\_of\_red))}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-86-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 5: Generate three matrices corresponding to Red, Green, and Blue values}
\NormalTok{red }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(shades\_of\_red, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}
\NormalTok{green }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(shades\_of\_green, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}
\NormalTok{blue }\OtherTok{=} \FunctionTok{matrix}\NormalTok{(shades\_of\_blue, }\AttributeTok{nrow =}\NormalTok{ height, }\AttributeTok{ncol =}\NormalTok{ width)}

\CommentTok{\# Step 6: Generate an array by combining the three matrices}
\NormalTok{image.array }\OtherTok{=} \FunctionTok{array}\NormalTok{(}\FunctionTok{c}\NormalTok{(red, green, blue), }\AttributeTok{dim =} \FunctionTok{c}\NormalTok{(width, height, layers))}
\FunctionTok{dim}\NormalTok{(image.array)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 300 300   3
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 7: Create RasterBrick}
\NormalTok{image }\OtherTok{=} \FunctionTok{brick}\NormalTok{(image.array)}
\FunctionTok{print}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## class      : RasterBrick 
## dimensions : 300, 300, 90000, 3  (nrow, ncol, ncell, nlayers)
## resolution : 0.003333, 0.003333  (x, y)
## extent     : 0, 1, 0, 1  (xmin, xmax, ymin, ymax)
## crs        : NA 
## source     : memory
## names      : layer.1, layer.2, layer.3 
## min values :       0,       0,       0 
## max values :     255,     255,     255
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Step 8: Plot RGB}
\FunctionTok{plotRGB}\NormalTok{(image)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning in .couldBeLonLat(x, warnings = warnings): CRS is NA.
## Assuming it is longitude/latitude
\end{verbatim}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-86-2.pdf}

From the examples above, you recognize that in order to generate/manipulate raster images in R, we basically generate/manipulate matrices/arrays (i.e., structures very common to any data analytics task in R). Several additional R packages come with pre-defined functions for more advanced image manipulations in R.

\hypertarget{advanced-image-manipulation-using-imagemagick}{%
\subsection{Advanced Image Manipulation using ImageMagick}\label{advanced-image-manipulation-using-imagemagick}}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load package}
\FunctionTok{library}\NormalTok{(magick)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Linking to ImageMagick 6.9.11.60
## Enabled features: fontconfig, freetype, fftw, heic, lcms, pango, webp, x11
## Disabled features: cairo, ghostscript, raw, rsvg
\end{verbatim}

\begin{verbatim}
## Using 4 threads
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# We can import images from the web}
\NormalTok{frink }\OtherTok{\textless{}{-}} \FunctionTok{image\_read}\NormalTok{(}\StringTok{"https://jeroen.github.io/images/frink.png"}\NormalTok{)}
\FunctionTok{print}\NormalTok{(frink)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 1 x 7
##   format width height colorspace matte filesize density
##   <chr>  <int>  <int> <chr>      <lgl>    <int> <chr>  
## 1 PNG      220    445 sRGB       TRUE     73494 72x72
\end{verbatim}

\includegraphics[width=3.06in]{datahandling_files/figure-latex/unnamed-chunk-87-1}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Rotate}
\FunctionTok{image\_rotate}\NormalTok{(frink, }\DecValTok{45}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=6.56in]{datahandling_files/figure-latex/unnamed-chunk-87-2}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Flip}
\FunctionTok{image\_flip}\NormalTok{(frink)}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=3.06in]{datahandling_files/figure-latex/unnamed-chunk-87-3}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Flop}
\FunctionTok{image\_flop}\NormalTok{(frink)}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=3.06in]{datahandling_files/figure-latex/unnamed-chunk-87-4}

\hypertarget{optical-character-recognition}{%
\subsection{Optical Character Recognition}\label{optical-character-recognition}}

A common context to encounter image data in empirical economic research is the digitization of old texts. In that context, oprical character recognition (OCR) is used to extract text from scanned images. For example, in a setting where an archive of old newspapers is digitized in order to analyze historical news reports. R provides a straightforward approach to OCR in which the input is an image file (e.g., a png-file) and the output is a character string.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# For Optical Character Recognition}
\FunctionTok{library}\NormalTok{(tesseract)}

\NormalTok{img }\OtherTok{\textless{}{-}} \FunctionTok{image\_read}\NormalTok{(}\StringTok{"https://s3.amazonaws.com/libapps/accounts/30502/images/new\_york\_times.png"}\NormalTok{)}
\FunctionTok{print}\NormalTok{(img)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 1 x 7
##   format width height colorspace matte filesize density
##   <chr>  <int>  <int> <chr>      <lgl>    <int> <chr>  
## 1 PNG      806    550 sRGB       FALSE   714189 38x38
\end{verbatim}

\includegraphics[width=11.19in]{datahandling_files/figure-latex/unnamed-chunk-88-1}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{headline }\OtherTok{\textless{}{-}} 
  \FunctionTok{image\_crop}\NormalTok{(}\AttributeTok{image =}\NormalTok{ img, }\AttributeTok{geometry =} \StringTok{\textquotesingle{}806x180\textquotesingle{}}\NormalTok{)}

\NormalTok{headline}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=11.19in]{datahandling_files/figure-latex/unnamed-chunk-88-2}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Extract text}
\NormalTok{headline\_text }\OtherTok{\textless{}{-}} \FunctionTok{image\_ocr}\NormalTok{(headline)}
\FunctionTok{cat}\NormalTok{(headline\_text)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## The New Work Times. ==S=
## 
## TITANIC SINKS FOUR HOURS AFTER HITTING ICEBERG;
## 866 RESCUED BY CARPATHIA, PROBABLY 1250 PERISH;
## ISMAY SAFE, MRS. ASTOR MAYBE, NOTED NAMES MISSING
\end{verbatim}

\hypertarget{vector-images-in-r}{%
\section{Vector Images in R}\label{vector-images-in-r}}

An alternative to storing figures in matrix/bitmap-based image-files are vector-based graphics. Vector-based formats are not stored in the form of arrays/matrices that contain the color information of each pixel of an image. Instead they define the shapes, colors and coordinates of the objects shown in an image. Typically such vector-based images are computer drawings, plots, maps, and blueprints of technical infstructure (and not, for example, photos). There are different specific file-formats to store vector-based graphics, but typically they use a nesting structure and basic syntax that is similar to or a version of XML. That is, in order to work with the basic data contained in such files, we often can use familiar functions like \texttt{read\_xml()}. To translate the basic vector-image data into images and modify these images additional packages such as \texttt{magick} are available. The following code example demonstrates these points. We first import the raw vector-image data of the R logo (stored as a SVG-file) as a XML-file into R. This allows us to have a closer look at the underlying data structure of such files. Then, we import it as an actual vector-based graphic via \texttt{image\_read\_svg()}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Common Packages for Vector Files}
\FunctionTok{library}\NormalTok{(magick)}

\CommentTok{\# Download and read svg image from url}
\NormalTok{URL }\OtherTok{\textless{}{-}} \StringTok{"https://upload.wikimedia.org/wikipedia/commons/1/1b/R\_logo.svg"}
\NormalTok{Rlogo\_xml }\OtherTok{\textless{}{-}} \FunctionTok{read\_xml}\NormalTok{(URL)}

\CommentTok{\# Data structure}
\NormalTok{Rlogo\_xml }
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## {xml_document}
## <svg preserveAspectRatio="xMidYMid" width="724" height="561" viewBox="0 0 724 561" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
## [1] <defs>\n  <linearGradient id="gradientFill-1" x1="0" x2="1 ...
## [2] <path d="M361.453,485.937 C162.329,485.937 0.906,377.828 0 ...
## [3] <path d="M550.000,377.000 C550.000,377.000 571.822,383.585 ...
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{xml\_structure}\NormalTok{(Rlogo\_xml)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## <svg [preserveAspectRatio, width, height, viewBox, xmlns, xmlns:xlink]>
##   <defs>
##     <linearGradient [id, x1, x2, y1, y2, gradientUnits, spreadMethod]>
##       <stop [offset, stop-color, stop-opacity]>
##       <stop [offset, stop-color, stop-opacity]>
##     <linearGradient [id, x1, x2, y1, y2, gradientUnits, spreadMethod]>
##       <stop [offset, stop-color, stop-opacity]>
##       <stop [offset, stop-color, stop-opacity]>
##   <path [d, fill, fill-rule]>
##   <path [d, fill, fill-rule]>
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Raw data}
\NormalTok{Rlogo\_text }\OtherTok{\textless{}{-}} \FunctionTok{as.character}\NormalTok{(Rlogo\_xml)}

\CommentTok{\# Plot}
\NormalTok{svg\_img }\OtherTok{=} \FunctionTok{image\_read\_svg}\NormalTok{(Rlogo\_text)}
\FunctionTok{image\_info}\NormalTok{(svg\_img)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 1 x 7
##   format width height colorspace matte filesize density
##   <chr>  <int>  <int> <chr>      <lgl>    <int> <chr>  
## 1 PNG      724    561 sRGB       TRUE         0 72x72
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{svg\_img}
\end{Highlighting}
\end{Shaded}

\begin{center}\includegraphics[width=0.15\linewidth]{img/R_logo.svg} \end{center}

\hypertarget{data-sources-data-gathering-data-import}{%
\chapter{Data Sources, Data Gathering, Data Import}\label{data-sources-data-gathering-data-import}}

In this chapter, we put the key concepts learned so far (text files for data storage, parsers, encoding, data structures) together and apply them to master the first key bottleneck in the data pipeline: how to \emph{import} raw data from various sources and \emph{export/store} them for further processing in the pipeline.

\hypertarget{sourcesformats}{%
\section{Sources/formats}\label{sourcesformats}}

In the previous chapters, we learned how data is stored in text files and how different data structures/formats/syntaxes help to organize the data in these files. Along the way, we have encountered key data formats that are used in various settings to store and transfer data:

\begin{itemize}
\tightlist
\item
  CSV (typical for rectangular/table-like data)
\item
  Variants of CSV (tab-delimited, fix length, etc.)
\item
  XML and JSON (useful for complex/high-dimensional data sets)
\item
  HTML (a markup language to define the structure and layout of webpages)
\item
  Unstructured text
\end{itemize}

Depending on the \emph{data source}, data might come in one or the other form. With the increasing importance of the Internet as a data source for economic research, properly handling XML, JSON, and HTML is becoming more important. However, in applied economic research, various other formats can be encountered:

\begin{itemize}
\tightlist
\item
  Excel spreadsheets (\texttt{.xls})
\item
  Formats specific to statistical software packages (SPSS: \texttt{.sav}, STATA: \texttt{.dat}, etc.)
\item
  Built-in R datasets
\item
  Binary formats
\end{itemize}

While we will cover/revisit how to import all of these formats here, it is important to keep in mind that the learned fundamental concepts are as important (or even more important) than knowing which function to call in R for each of these cases. New formats might evolve and become more relevant in the future for which no R function yet exists. However, the underlying logic of how formats to structure data work will hardly change.

\hypertarget{data-gathering-procedure}{%
\section{Data gathering procedure}\label{data-gathering-procedure}}

Before we set out to gather/import data from diverse sources, we should start organizing the procedure in an R script. This script will be the beginning of our pipeline!

First, open a new R script in RStudio and save it as \texttt{import\_data.R} in your \texttt{code} folder. Take your time to meaningfully describe what the script is all about in the first few lines of the file:

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#}
\CommentTok{\# Data Handling Course: Example Script for Data Gathering and Import}
\CommentTok{\#}
\CommentTok{\# Imports data from ...}
\CommentTok{\# Input: links to data sources (data comes in ... format)}
\CommentTok{\# Output: cleaned data as CSV}
\CommentTok{\#}
\CommentTok{\# U. Matter, St.Gallen, 2019}
\DocumentationTok{\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#}
\end{Highlighting}
\end{Shaded}

RStudio recognizes different sections in a script, whereby section headers are indicated by \texttt{-\/-\/-\/-\/-\/-\/-\/-\/-\/-}. This helps to organize the script into different tasks further. Usually, it makes sense to start with a `meta' section in which all necessary packages are loaded and fix variables initiated.

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#}
\CommentTok{\# Data Handling Course: Example Script for Data Gathering and Import}
\CommentTok{\#}
\CommentTok{\# Imports data from ...}
\CommentTok{\# Input: links to data sources (data comes in ... format)}
\CommentTok{\# Output: cleaned data as CSV}
\CommentTok{\#}
\CommentTok{\# U. Matter, St.Gallen, 2019}
\DocumentationTok{\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#}


\CommentTok{\# SET UP {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}
\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(tidyr)}

\CommentTok{\# set fix variables}
\NormalTok{INPUT\_PATH }\OtherTok{\textless{}{-}} \StringTok{"/rawdata"}
\NormalTok{OUTPUT\_FILE }\OtherTok{\textless{}{-}} \StringTok{"/final\_data/datafile.csv"}
\end{Highlighting}
\end{Shaded}

Finally, we add sections with the actual code (in the case of a data import script, maybe one section per data source).

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#}
\CommentTok{\# Project XY: Data Gathering and Import}
\CommentTok{\#}
\CommentTok{\# This script is the first part of the data pipeline of project XY.}
\CommentTok{\# It imports data from ...}
\CommentTok{\# Input: links to data sources (data comes in ... format)}
\CommentTok{\# Output: cleaned data as CSV}
\CommentTok{\#}
\CommentTok{\# U. Matter, St.Gallen, 2019}
\DocumentationTok{\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#\#}


\CommentTok{\# SET UP {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}
\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(tidyr)}

\CommentTok{\# set fix variables}
\NormalTok{INPUT\_PATH }\OtherTok{\textless{}{-}} \StringTok{"/rawdata"}
\NormalTok{OUTPUT\_FILE }\OtherTok{\textless{}{-}} \StringTok{"/final\_data/datafile.csv"}


\CommentTok{\# IMPORT RAW DATA FROM CSVs {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}
\end{Highlighting}
\end{Shaded}

\hypertarget{loadingimporting-rectangular-data}{%
\section[Loading/importing rectangular data]{\texorpdfstring{Loading/importing rectangular data\footnote{This section is based on \citet{umatter_2018}.}}{Loading/importing rectangular data}}\label{loadingimporting-rectangular-data}}

\hypertarget{loading-built-in-datasets}{%
\subsection{Loading built-in datasets}\label{loading-built-in-datasets}}

We start with the simplest case of loading/importing data. The basic R installation provides some example datasets to try out R's statistics functions. In the introduction to visualization techniques with R and the statistics examples in the chapters to come, we will rely on some of these datasets for simplicity. Note that the usage of these simple datasets shipped with basic R are very helpful when practicing/learning R on your own. Many R packages use these datasets over and over again in their documentation and examples. Moreover, extensive documentations and tutorials online also use these datasets (see for example the \href{https://ggplot2.tidyverse.org/}{ggplot2 documentation}). And, they are very useful when searching help on \href{https://stackoverflow.com/questions/tagged/r}{Stackoverflow} in the context of data analysis/manipulation with R, as you should provide a code example based on some data that everybody can easily load and access.

In order to load such datasets, simply use the \texttt{data()}-function:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{data}\NormalTok{(swiss)}
\end{Highlighting}
\end{Shaded}

In this case, we load a dataset called \texttt{swiss}. After loading it, the data is stored in a variable of the same name as the dataset (here `\texttt{swiss}'). We can inspect it and have a look at the first few rows:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# inspect the structure}
\FunctionTok{str}\NormalTok{(swiss)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 'data.frame':    47 obs. of  6 variables:
##  $ Fertility       : num  80.2 83.1 92.5 85.8 76.9 76.1 83.8 92.4 82.4 82.9 ...
##  $ Agriculture     : num  17 45.1 39.7 36.5 43.5 35.3 70.2 67.8 53.3 45.2 ...
##  $ Examination     : int  15 6 5 12 17 9 16 14 12 16 ...
##  $ Education       : int  12 9 5 7 15 7 7 8 7 13 ...
##  $ Catholic        : num  9.96 84.84 93.4 33.77 5.16 ...
##  $ Infant.Mortality: num  22.2 22.2 20.2 20.3 20.6 26.6 23.6 24.9 21 24.4 ...
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# look at the first few rows}
\FunctionTok{head}\NormalTok{(swiss)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##              Fertility Agriculture Examination Education
## Courtelary        80.2        17.0          15        12
## Delemont          83.1        45.1           6         9
## Franches-Mnt      92.5        39.7           5         5
## Moutier           85.8        36.5          12         7
## Neuveville        76.9        43.5          17        15
## Porrentruy        76.1        35.3           9         7
##              Catholic Infant.Mortality
## Courtelary       9.96             22.2
## Delemont        84.84             22.2
## Franches-Mnt    93.40             20.2
## Moutier         33.77             20.3
## Neuveville       5.16             20.6
## Porrentruy      90.57             26.6
\end{verbatim}

To get a list of all the built-in datasets, type \texttt{data()} into the console and hit enter. To get more information about a given dataset, use the help function (e.g., \texttt{?swiss})

\hypertarget{importing-rectangular-data-from-text-files}{%
\subsection{Importing rectangular data from text-files}\label{importing-rectangular-data-from-text-files}}

In most cases of applying R for data analysis, students and researchers rely on importing data from files stored on the hard disk. Typically, such datasets are stored in a text file format such as `Comma Separated Values' (CSV). In economics, one also frequently encounters data stored in specific formats of commercial statistics/data analysis packages such as SPSS or STATA. Moreover, when collecting data on your own, you might rely on a spreadsheet tool like Microsoft Excel. Data from all these formats can easily be imported into R (in some cases, additional packages have to be loaded, though). Thereby, what happens `under the hood' is essentially the same for all of these formats. Somebody has implemented the respective \emph{parser} as an R function that accepts a character string with the path or URL to the data source as input.

\hypertarget{comma-separated-values-csv}{%
\subsubsection{Comma Separated Values (CSV)}\label{comma-separated-values-csv}}

Recall how in this format, data values of one observation are stored in one row of a text file, while commas separate the variables/columns. For example, the following code block shows how the first two rows of the \texttt{swiss}-dataset would look like when stored in a CSV:

\begin{verbatim}
"District","Fertility","Agriculture","Examination","Education","Catholic","Infant.Mortality"
"Courtelary",80.2,17,15,12,9.96,22.2
\end{verbatim}

The function \texttt{read.csv()} imports such files from disk into R (in the form of a \texttt{data\ frame}). In this example, the \texttt{swiss}-dataset is stored locally on our disk in the folder \texttt{data}:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{swiss\_imported }\OtherTok{\textless{}{-}} \FunctionTok{read.csv}\NormalTok{(}\StringTok{"data/swiss.csv"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

Alternatively, we could use the newer \texttt{read\_csv()} function, which would return a \texttt{tibble}.

\hypertarget{spreadsheetsexcel}{%
\subsubsection{Spreadsheets/Excel}\label{spreadsheetsexcel}}

To read excel spreadsheets, we need to install an additional R package called \texttt{readxl}.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# install the package}
\FunctionTok{install.packages}\NormalTok{(}\StringTok{"readxl"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

Then we load this additional package (`library') and use the package's \texttt{read\_excel()}-function to import data from an excel-sheet. In the example below, the same data as above is stored in an excel-sheet called \texttt{swiss.xlsx}, again in a folder called \texttt{data}.

\begin{verbatim}
## New names:
## * `` -> `...1`
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load the package}
\FunctionTok{library}\NormalTok{(readxl)}

\CommentTok{\# import data from a spreadsheet}
\NormalTok{swiss\_imported }\OtherTok{\textless{}{-}} \FunctionTok{read\_excel}\NormalTok{(}\StringTok{"data/swiss.xlsx"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\hypertarget{data-from-other-data-analysis-software}{%
\subsubsection{Data from other data analysis software}\label{data-from-other-data-analysis-software}}

The R packages \texttt{foreign} and \texttt{haven} contain functions to import data from formats used in other statistics/data analysis software, such as SPSS and STATA.

In the following example we use \texttt{haven}`s \texttt{read\_spss()} function to import a version of the \texttt{swiss}-dataset stored in SPSS' \texttt{.sav}-format (again stored in the folder called \texttt{data}).

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# install the package (if not yet installed):}
\CommentTok{\# install.packages("haven")}

\CommentTok{\# load the package}
\FunctionTok{library}\NormalTok{(haven)}

\CommentTok{\# read the data}
\NormalTok{swiss\_imported }\OtherTok{\textless{}{-}} \FunctionTok{read\_spss}\NormalTok{(}\StringTok{"data/swiss.sav"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\hypertarget{import-and-parse-with-readr}{%
\section[Import and parse with \texttt{readr}]{\texorpdfstring{Import and parse with \texttt{readr}\footnote{This is a summary of Chapter 8 in \citet{wickham_grolemund2017}.}}{Import and parse with readr}}\label{import-and-parse-with-readr}}

The \texttt{readr} package is automatically installed and loaded with the installation/loading of \texttt{tidyverse}. It provides a set of functions to read different types of rectangular data formats and is usually more robust and faster than similar functions in the basic R distribution. Each of these functions expects either a character string with a path pointing to a file or a character string directly containing the data.

\hypertarget{basic-usage-of-readr-functions}{%
\subsection{\texorpdfstring{Basic usage of \texttt{readr} functions}{Basic usage of readr functions}}\label{basic-usage-of-readr-functions}}

For example, we can parse the first lines of the swiss dataset directly like this.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{library}\NormalTok{(readr)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Attaching package: 'readr'
\end{verbatim}

\begin{verbatim}
## The following object is masked from 'package:rvest':
## 
##     guess_encoding
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{read\_csv}\NormalTok{(}\StringTok{\textquotesingle{}"District","Fertility","Agriculture","Examination","Education","Catholic","Infant.Mortality"}
\StringTok{"Courtelary",80.2,17,15,12,9.96,22.2\textquotesingle{}}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Rows: 1 Columns: 7
\end{verbatim}

\begin{verbatim}
## -- Column specification -----------------------------------------
## Delimiter: ","
## chr (1): District
## dbl (6): Fertility, Agriculture, Examination, Education, Cath...
## 
## i Use `spec()` to retrieve the full column specification for this data.
## i Specify the column types or set `show_col_types = FALSE` to quiet this message.
\end{verbatim}

\begin{verbatim}
## # A tibble: 1 x 7
##   District   Fertility Agriculture Examination Education Catholic
##   <chr>          <dbl>       <dbl>       <dbl>     <dbl>    <dbl>
## 1 Courtelary      80.2          17          15        12     9.96
## # ... with 1 more variable: Infant.Mortality <dbl>
\end{verbatim}

or read the entire \texttt{swiss} dataset by pointing to the file

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{swiss }\OtherTok{\textless{}{-}} \FunctionTok{read\_csv}\NormalTok{(}\StringTok{"data/swiss.csv"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Rows: 47 Columns: 7
## -- Column specification -----------------------------------------
## Delimiter: ","
## chr (1): District
## dbl (6): Fertility, Agriculture, Examination, Education, Cath...
## 
## i Use `spec()` to retrieve the full column specification for this data.
## i Specify the column types or set `show_col_types = FALSE` to quiet this message.
\end{verbatim}

In either case, the result is a \texttt{tibble}:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{swiss}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 47 x 7
##    District  Fertility Agriculture Examination Education Catholic
##    <chr>         <dbl>       <dbl>       <dbl>     <dbl>    <dbl>
##  1 Courtela~      80.2        17            15        12     9.96
##  2 Delemont       83.1        45.1           6         9    84.8 
##  3 Franches~      92.5        39.7           5         5    93.4 
##  4 Moutier        85.8        36.5          12         7    33.8 
##  5 Neuvevil~      76.9        43.5          17        15     5.16
##  6 Porrentr~      76.1        35.3           9         7    90.6 
##  7 Broye          83.8        70.2          16         7    92.8 
##  8 Glane          92.4        67.8          14         8    97.2 
##  9 Gruyere        82.4        53.3          12         7    97.7 
## 10 Sarine         82.9        45.2          16        13    91.4 
## # ... with 37 more rows, and 1 more variable:
## #   Infant.Mortality <dbl>
\end{verbatim}

The other \texttt{readr} functions have practically the same syntax and behavior. They are used for fixed-width files or CSV-type files with other delimiters than commas.

\hypertarget{parsing-and-data-types}{%
\subsection{Parsing and data types}\label{parsing-and-data-types}}

From inspecting the \texttt{swiss} tibble pointed out above, we recognize that \texttt{read\_csv} not only correctly recognizes observations and columns (parses the CSV correctly) but also automatically guesses the data type of the values in each column. The first column is of type double, the second one of type integer, etc. That is, \texttt{read\_csv} also parses each column-vector of the data set with the aim of recognizing which data type it is. For example, the data value \texttt{"12:00"} could be interpreted simply as a character string. Alternatively, it could also be interpreted as a \texttt{time} format.

If \texttt{"12:00"} is an element of the vector \texttt{c("12:00",\ "midnight",\ "noon")} it must be interpreted as a character string. If however it is an element of the vector \texttt{c("12:00",\ "14:30",\ "20:01")} we probably want R to import this as a \texttt{time} format. Now, how can \texttt{readr} handle the two cases? In simple terms, the package first guesses for each column vector which type is most appropriate. Then, it uses a couple of lower-level parsing functions (one written for each possible data type) in order to parse each column according to the respective guessed type. We can demonstrate this for the two example vectors above.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{read\_csv}\NormalTok{(}\StringTok{\textquotesingle{}A,B}
\StringTok{         12:00, 12:00}
\StringTok{         14:30, midnight}
\StringTok{         20:01, noon\textquotesingle{}}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Rows: 3 Columns: 2
## -- Column specification -----------------------------------------
## Delimiter: ","
## chr  (1): B
## time (1): A
## 
## i Use `spec()` to retrieve the full column specification for this data.
## i Specify the column types or set `show_col_types = FALSE` to quiet this message.
\end{verbatim}

\begin{verbatim}
## # A tibble: 3 x 2
##   A      B       
##   <time> <chr>   
## 1 12:00  12:00   
## 2 14:30  midnight
## 3 20:01  noon
\end{verbatim}

Under the hood \texttt{read\_csv()} used the \texttt{guess\_parser()}- function to determine which type the two vectors likely contain:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{guess\_parser}\NormalTok{(}\FunctionTok{c}\NormalTok{(}\StringTok{"12:00"}\NormalTok{, }\StringTok{"midnight"}\NormalTok{, }\StringTok{"noon"}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "character"
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{guess\_parser}\NormalTok{(}\FunctionTok{c}\NormalTok{(}\StringTok{"12:00"}\NormalTok{, }\StringTok{"14:30"}\NormalTok{, }\StringTok{"20:01"}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "time"
\end{verbatim}

\hypertarget{importing-web-data-formats}{%
\section{Importing web data formats}\label{importing-web-data-formats}}

\hypertarget{xml-in-r}{%
\subsection[XML in R]{\texorpdfstring{XML in R\footnote{This section is based on \citet{umatter_2018b}.}}{XML in R}}\label{xml-in-r}}

There are several XML-parsers already implemented in R packages specifically written for working with XML data. Thus, we do not have to understand the XML syntax in every detail to work with this data format in R. The already familiar package \texttt{xml2} (automatically loaded when loading \texttt{rvest}) provides the \texttt{read\_xml()} function which we can use to read the exemplary XML-document.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(xml2)}

\CommentTok{\# parse XML, represent XML document as R object}
\NormalTok{xml\_doc }\OtherTok{\textless{}{-}} \FunctionTok{read\_xml}\NormalTok{(}\StringTok{"data/customers.xml"}\NormalTok{)}
\NormalTok{xml\_doc}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## {xml_document}
## <customers>
## [1] <person>\n  <name>John Doe</name>\n  <orders>\n    <produc ...
## [2] <person>\n  <name>Peter Pan</name>\n  <orders>\n    <produ ...
\end{verbatim}

The same package also has various functions to access, extract, and manipulate data from a parsed XML document. In the following code example, we have a look at the most useful functions for our purposes (see the package's \href{https://cran.r-project.org/web/packages/xml2/vignettes/modification.html}{vignette} for more details).

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# navigate through the XML document (recall the tree{-}like nested structure similar to HTML)}
\CommentTok{\# navigate downwards}
\CommentTok{\# \textquotesingle{}customers\textquotesingle{} is the root node, persons are ‘their children\textquotesingle{}}
\NormalTok{persons }\OtherTok{\textless{}{-}} \FunctionTok{xml\_children}\NormalTok{(xml\_doc)}
\CommentTok{\# navigate sidewards}
\FunctionTok{xml\_siblings}\NormalTok{(persons)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## {xml_nodeset (2)}
## [1] <person>\n  <name>Peter Pan</name>\n  <orders>\n    <produ ...
## [2] <person>\n  <name>John Doe</name>\n  <orders>\n    <produc ...
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# navigate upwards}
\FunctionTok{xml\_parents}\NormalTok{(persons)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## {xml_nodeset (1)}
## [1] <customers>\n  <person>\n    <name>John Doe</name>\n    <o ...
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# find data via XPath}
\NormalTok{customer\_names }\OtherTok{\textless{}{-}} \FunctionTok{xml\_find\_all}\NormalTok{(xml\_doc, }\AttributeTok{xpath =} \StringTok{".//name"}\NormalTok{)}
\CommentTok{\# extract the data as text}
\FunctionTok{xml\_text}\NormalTok{(customer\_names)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "John Doe"  "Peter Pan"
\end{verbatim}

\hypertarget{json-in-r}{%
\subsection[JSON in R]{\texorpdfstring{JSON in R\footnote{This section is based on \citet{umatter_2018b}.}}{JSON in R}}\label{json-in-r}}

Again, we can rely on an R package (\texttt{jsonlite}) providing high-level functions to read, manipulate, and extract data when working with JSON documents in R. An important difference between working with XML- and HTML-documents is that XPath is not compatible with JSON. However, as \texttt{jsonlite} represents parsed JSON as R objects of class \texttt{list} and/or \texttt{data-frame}, we can work with the parsed document as with any other R-object of the same class. The following example illustrates this point.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(jsonlite)}

\CommentTok{\# parse the JSON document shown in the example above}
\NormalTok{json\_doc }\OtherTok{\textless{}{-}} \FunctionTok{fromJSON}\NormalTok{(}\StringTok{"data/person.json"}\NormalTok{)}

\CommentTok{\# look at the structure of the document}
\FunctionTok{str}\NormalTok{(json\_doc)}

\CommentTok{\# navigate the nested lists, extract data}
\CommentTok{\# extract the address part}
\NormalTok{json\_doc}\SpecialCharTok{$}\NormalTok{address}
\CommentTok{\# extract the gender (type)}
\NormalTok{json\_doc}\SpecialCharTok{$}\NormalTok{gender}\SpecialCharTok{$}\NormalTok{type}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## List of 6
##  $ firstName  : chr "John"
##  $ lastName   : chr "Smith"
##  $ age        : int 25
##  $ address    :List of 4
##   ..$ streetAddress: chr "21 2nd Street"
##   ..$ city         : chr "New York"
##   ..$ state        : chr "NY"
##   ..$ postalCode   : chr "10021"
##  $ phoneNumber:'data.frame': 2 obs. of  2 variables:
##   ..$ type  : chr [1:2] "home" "fax"
##   ..$ number: chr [1:2] "212 555-1234" "646 555-4567"
##  $ gender     :List of 1
##   ..$ type: chr "male"
\end{verbatim}

\begin{verbatim}
## $streetAddress
## [1] "21 2nd Street"
## 
## $city
## [1] "New York"
## 
## $state
## [1] "NY"
## 
## $postalCode
## [1] "10021"
\end{verbatim}

\begin{verbatim}
## [1] "male"
\end{verbatim}

\hypertarget{tutorial-advanced-importing-data-from-a-html-table-on-a-website}{%
\subsection{Tutorial (advanced): Importing data from a HTML table (on a website)}\label{tutorial-advanced-importing-data-from-a-html-table-on-a-website}}

In the chapter on high-dimensional data, we discussed the \emph{Hypertext Markup Language (HTML)} as code to define the structure/content of a website and HTML-documents as semi-structured data sources. The following tutorial revisits the basic steps in importing data from an HTML table into R.

The aim of the tutorial is to generate a CSV file containing data on \href{https://en.wikipedia.org/wiki/Divided_government}{`divided government'} in US politics. We use the following Wikipedia page as a data source: \url{https://en.wikipedia.org/wiki/Divided_government_in_the_United_States}. The page contains a table indicating the president's party, and the majority party in the US House and the US Senate per Congress (2-year periods). The first few rows of the cleaned data are supposed to look like this:

\begin{verbatim}
##    year  president senate house
## 1: 1861    Lincoln      R     R
## 2: 1862    Lincoln      R     R
## 3: 1863    Lincoln      R     R
## 4: 1864    Lincoln      R     R
## 5: 1865 A. Johnson      R     R
## 6: 1866 A. Johnson      R     R
\end{verbatim}

In a first step, we initiate fix variables for paths and load additional R packages needed to handle data stored in HTML documents.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# SETUP {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(rvest)}
\FunctionTok{library}\NormalTok{(data.table)}

\CommentTok{\# fix vars}
\NormalTok{SOURCE\_PATH }\OtherTok{\textless{}{-}} \StringTok{"https://en.wikipedia.org/wiki/Divided\_government\_in\_the\_United\_States"}
\NormalTok{OUTPUT\_PATH }\OtherTok{\textless{}{-}} \StringTok{"data/divided\_gov.csv"}
\end{Highlighting}
\end{Shaded}

Now we write the part of the script that fetches the data from the Web. This part consists of three steps. First we fetch the entire website (HTML document) from Wikipedia with (\texttt{read\_html()}). Second, we extract the part of the website containing the table with the data we want (via \texttt{html\_node()}). Finally, we parse the HTML table and store its content in a data frame called \texttt{tab.} The last line of the code chunk below removes the last row of the data frame (you can see on the website that this row is not needed)

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# FETCH/FORMAT DATA {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\CommentTok{\# fetch from web}
\NormalTok{doc }\OtherTok{\textless{}{-}} \FunctionTok{read\_html}\NormalTok{(SOURCE\_PATH)}
\NormalTok{tab }\OtherTok{\textless{}{-}} \FunctionTok{html\_table}\NormalTok{(doc,}\AttributeTok{fill=}\ConstantTok{TRUE}\NormalTok{)[[}\DecValTok{2}\NormalTok{]]}
\NormalTok{tab }\OtherTok{\textless{}{-}}\NormalTok{ tab[}\SpecialCharTok{{-}}\FunctionTok{nrow}\NormalTok{(tab), ] }\CommentTok{\# remove last row (not containing data)}
\end{Highlighting}
\end{Shaded}

Now we clean the data to get a data set more suitable for data analysis. Note that the original table contains information per congress (2-year periods). However, as the sample above shows, we aim for a panel at the year level. The following code iterates through the rows of the data frame and generates for each row per congress several two rows (one for each year in the congress).\footnote{See \texttt{?strsplit}, \texttt{?unlist}, and \href{https://www.oreilly.com/content/an-introduction-to-regular-expressions/}{this introduction to regular expressions} for the background of how this is done in the code example here.}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# generate year{-}level data. frame}

\CommentTok{\# prepare loop}
\NormalTok{all\_years }\OtherTok{\textless{}{-}} \FunctionTok{list}\NormalTok{() }\CommentTok{\# the container}
\NormalTok{n }\OtherTok{\textless{}{-}} \FunctionTok{length}\NormalTok{(tab}\SpecialCharTok{$}\NormalTok{Year) }\CommentTok{\# number of cases to iterate through}
\FunctionTok{length}\NormalTok{(all\_years) }\OtherTok{\textless{}{-}}\NormalTok{ n}
\CommentTok{\# generate year{-}level observations. row by row.}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in} \DecValTok{1}\SpecialCharTok{:}\NormalTok{n)\{}
      \CommentTok{\# select row}
\NormalTok{      row }\OtherTok{\textless{}{-}}\NormalTok{ tab[i,]}
\NormalTok{      y }\OtherTok{\textless{}{-}}\NormalTok{ row}\SpecialCharTok{$}\NormalTok{Year}
      \CommentTok{\#}
\NormalTok{      begin }\OtherTok{\textless{}{-}} \FunctionTok{as.numeric}\NormalTok{(}\FunctionTok{unlist}\NormalTok{(}\FunctionTok{strsplit}\NormalTok{(}\AttributeTok{x =}\NormalTok{ y, }\AttributeTok{split =} \StringTok{"[}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{–}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{{-}]"}\NormalTok{, }\AttributeTok{perl =} \ConstantTok{TRUE}\NormalTok{))[}\DecValTok{1}\NormalTok{])}
\NormalTok{      end }\OtherTok{\textless{}{-}} \FunctionTok{as.numeric}\NormalTok{(}\FunctionTok{unlist}\NormalTok{(}\FunctionTok{strsplit}\NormalTok{(}\AttributeTok{x =}\NormalTok{ y, }\AttributeTok{split =}  \StringTok{"[}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{–}\SpecialCharTok{\textbackslash{}\textbackslash{}}\StringTok{{-}]"}\NormalTok{))[}\DecValTok{2}\NormalTok{])}
\NormalTok{      tabnew }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{year=}\NormalTok{begin}\SpecialCharTok{:}\NormalTok{(end}\DecValTok{{-}1}\NormalTok{), }\AttributeTok{president=}\NormalTok{row}\SpecialCharTok{$}\NormalTok{President, }\AttributeTok{senate=}\NormalTok{row}\SpecialCharTok{$}\NormalTok{Senate, }\AttributeTok{house=}\NormalTok{row}\SpecialCharTok{$}\NormalTok{House)}
\NormalTok{      all\_years[[i]] }\OtherTok{\textless{}{-}}\NormalTok{ tabnew }\CommentTok{\# store in container}
\NormalTok{\}}

\CommentTok{\# stack all rows together}
\NormalTok{allyears }\OtherTok{\textless{}{-}} \FunctionTok{bind\_rows}\NormalTok{(all\_years)}
\end{Highlighting}
\end{Shaded}

In a last step, we inspect the collected data and write it to a CSV file.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# WRITE TO DISK {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\CommentTok{\# inspect}
\FunctionTok{head}\NormalTok{(allyears)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   year  president senate house
## 1 1861    Lincoln      R     R
## 2 1862    Lincoln      R     R
## 3 1863    Lincoln      R     R
## 4 1864    Lincoln      R     R
## 5 1865 A. Johnson      R     R
## 6 1866 A. Johnson      R     R
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# write to CSV}
\FunctionTok{write\_csv}\NormalTok{(allyears, }\AttributeTok{file=}\NormalTok{OUTPUT\_PATH)}
\end{Highlighting}
\end{Shaded}

\hypertarget{data-preparation}{%
\chapter{Data Preparation}\label{data-preparation}}

Importing a dataset properly is just the first of several milestones until an analysis-ready dataset is generated. In some cases, cleaning the raw data is a necessary step to facilitate/enable proper parsing of the data set to import it. However, most of the cleaning/preparation (`wrangling') with the data follows after properly parsing structured data. Many aspects of data wrangling are specific to certain datasets and an entire curriculum could be filled with different approaches and tools to address specific problems. Moreover, proficiency in data wrangling is generally a matter of experience in working with data, gained over many years. Here, we focus on two quite general and broadly applicable techniques that are central to cleaning and preparing a dataset for analysis: Simple string operations (find/replace parts of text strings) and reshaping rectangular data (wide to long/long to wide). The former is focused on individual variables at a time, while the latter typically happens at the level of the entire dataset.

\hypertarget{cleaning-data-with-basic-string-operations}{%
\section{Cleaning data with basic string operations}\label{cleaning-data-with-basic-string-operations}}

Recall that most of the data we read into R for analytic purposes is a collection of raw text (structured with special characters). When parsing the data to read it into R with high-level functions such as the ones provided in the \texttt{readr}-package, both the structure and the data types are considered. The resulting \texttt{data.frame}/\texttt{tibble} might thus contain variables (different columns) of type \texttt{character}, \texttt{factor}, or \texttt{integer}, etc. At this stage, it often happens that the raw data is not clean enough for the parser to recognize the data types in each column correctly, and it resorts to just parsing it as \texttt{character}. Indeed, if we have to deal with a very messy dataset, it can make a lot of sense to constrain the parser such that it reads each column a as \texttt{character}.

We first load this package as we rely on functions provided in the \texttt{tidyverse}.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{library}\NormalTok{(tidyverse)}
\end{Highlighting}
\end{Shaded}

Let's create a sample dataset to illustrate some of the typical issues regarding unclean data that we might encounter in empirical economic research (and many similar domains of data analysis).\footnote{The option \texttt{stringsAsFactors\ =\ FALSE} ensures that all of the columns in this data frame are of type \texttt{character}.}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{messy\_df }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{last\_name =} \FunctionTok{c}\NormalTok{(}\StringTok{"Wayne"}\NormalTok{, }\StringTok{"Trump"}\NormalTok{, }\StringTok{"Karl Marx"}\NormalTok{),}
                       \AttributeTok{first\_name =} \FunctionTok{c}\NormalTok{(}\StringTok{"John"}\NormalTok{, }\StringTok{"Melania"}\NormalTok{, }\StringTok{""}\NormalTok{),}
                       \AttributeTok{gender =} \FunctionTok{c}\NormalTok{(}\StringTok{"male"}\NormalTok{, }\StringTok{"female"}\NormalTok{, }\StringTok{"Man"}\NormalTok{),}
                       \AttributeTok{date =} \FunctionTok{c}\NormalTok{(}\StringTok{"2018{-}11{-}15"}\NormalTok{, }\StringTok{"2018.11.01"}\NormalTok{, }\StringTok{"2018/11/02"}\NormalTok{),}
                       \AttributeTok{income =} \FunctionTok{c}\NormalTok{(}\StringTok{"150,000"}\NormalTok{, }\StringTok{"250000"}\NormalTok{, }\StringTok{"10000"}\NormalTok{),}
                       \AttributeTok{stringsAsFactors =} \ConstantTok{FALSE}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

Assuming we have managed to read this dataset from a local file (with all columns as type \texttt{character}), the next step is to clean each of the columns such that the dataset is ready for analysis. Thereby we want to make sure that each variable (column) is set to a meaningful data type, once it is cleaned. The \emph{cleaning} of the parsed data is often easier to do when the data is of type \texttt{character}. Once it is cleaned, however, we can set it to a type that is more useful for the analysis part. For example, a column containing numeric values in the final dataset should be stored as \texttt{numeric} or \texttt{integer}, so we can perform math operations on it later on (compute sums, means, etc.).

\hypertarget{findreplace-character-strings-recode-factor-levels}{%
\subsection{Find/replace character strings, recode factor levels}\label{findreplace-character-strings-recode-factor-levels}}

Our dataset contains a typical categorical variable: \texttt{gender}. In R, storing such variables as type \texttt{factor} is good practice. Without really looking at the data values, we might thus be inclined to do the following:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{gender }\OtherTok{\textless{}{-}} \FunctionTok{as.factor}\NormalTok{(messy\_df}\SpecialCharTok{$}\NormalTok{gender)}
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{gender}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] male   female Man   
## Levels: female male Man
\end{verbatim}

The column is now of type \texttt{factor}. And we see that R defined the factor variable such that an observation can be one of three categories (`levels'): \texttt{female}, \texttt{male}, or \texttt{Man}. In terms of content, that probably does not make too much sense. If we were to analyze the data later and compute the sample's share of males, we would only count one instead of two. Hence, we better \emph{recode} the gender variable of male subjects as \texttt{male} and not \texttt{Man}. How can this be done programmatically?

One approach is to select all entries in \texttt{messy\_df\$gender} that are equal to \texttt{"Man"} and replace these entries with \texttt{"male"}.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{gender[messy\_df}\SpecialCharTok{$}\NormalTok{gender }\SpecialCharTok{==} \StringTok{"Man"}\NormalTok{] }\OtherTok{\textless{}{-}} \StringTok{"male"}
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{gender}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] male   female male  
## Levels: female male Man
\end{verbatim}

Note, however, that this approach is not perfect because R still considers \texttt{Man} as a valid possible category in this column. This can have consequences for certain analyses we might want to run on this dataset later on.\footnote{If we perform the same operation on this variable \emph{before} coercing it to a \texttt{factor}, this problem does not occur.} Alternatively, we can use a function \texttt{fct\_recode()} (provided in \texttt{tidyverse}), specifically for such operations with factors.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{gender }\OtherTok{\textless{}{-}} \FunctionTok{fct\_recode}\NormalTok{(messy\_df}\SpecialCharTok{$}\NormalTok{gender, }\StringTok{"male"} \OtherTok{=} \StringTok{"Man"}\NormalTok{)}
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{gender}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] male   female male  
## Levels: female male
\end{verbatim}

The latter can be very useful when several factor levels need to be recoded at once. Note that in both cases, the underlying logic is that we search for strings that are identical to \texttt{"Man"} and replace those values with \texttt{"male"}. Now, the gender variable is ready for analysis.

\hypertarget{removing-individual-characters-from-a-string}{%
\subsection{Removing individual characters from a string}\label{removing-individual-characters-from-a-string}}

The \texttt{income} column contains numbers, so let's try to set this column to type \texttt{integer}.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{as.integer}\NormalTok{(messy\_df}\SpecialCharTok{$}\NormalTok{income)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## Warning: NAs introduced by coercion
\end{verbatim}

\begin{verbatim}
## [1]     NA 250000  10000
\end{verbatim}

R is warning us that something did not go well when executing this code. We see that the first value of the original column has been replaced with \texttt{NA} (`Not Available'/`Not Applicable'/`No Answer'). The reason is that the original value contained a comma (\texttt{,}), a special character. The function \texttt{as\ integer()} does not know how to translate such a symbol to a number. Hence, the original data, value cannot be translated into a number (integer). In order to resolve this issue, we have to remove the comma (\texttt{,}) from this string. Or, more precisely, we will locate this specific character \emph{within} the string and replace it with an empty string (\texttt{""}) To do so, we'll use the function \texttt{str\_replace()} (for `string replace').

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{income }\OtherTok{\textless{}{-}} \FunctionTok{str\_replace}\NormalTok{(messy\_df}\SpecialCharTok{$}\NormalTok{income, }\AttributeTok{pattern =} \StringTok{","}\NormalTok{, }\AttributeTok{replacement =} \StringTok{""}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

Now we can successfully set the column as type integer.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{income }\OtherTok{\textless{}{-}} \FunctionTok{as.integer}\NormalTok{(messy\_df}\SpecialCharTok{$}\NormalTok{income)}
\end{Highlighting}
\end{Shaded}

\hypertarget{splitting-strings}{%
\subsection{Splitting strings}\label{splitting-strings}}

From looking at the \texttt{last\_name} and \texttt{first\_name} columns of our messy dataset, it becomes clear that the last row is not accurately coded. \texttt{Karl} should show up in the \texttt{first\_name} column. In order to correct this, we have to extract a part of one string and store this sub-string in another variable. There are several ways to do this. Here, it probably makes sense to split the original string into two parts, as the white space between \texttt{Karl} and \texttt{Marx} indicates the separation of first and last names. For this, we can use the function \texttt{str\_split()}.

First, we split the strings at every occurrence of white space (\texttt{"\ "}). Setting the option \texttt{simplify=TRUE}, we get a matrix containing the individual sub-strings after the splitting.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{splitnames }\OtherTok{\textless{}{-}} \FunctionTok{str\_split}\NormalTok{(messy\_df}\SpecialCharTok{$}\NormalTok{last\_name, }\AttributeTok{pattern =} \StringTok{" "}\NormalTok{, }\AttributeTok{simplify =} \ConstantTok{TRUE}\NormalTok{)}
\NormalTok{splitnames}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##      [,1]    [,2]  
## [1,] "Wayne" ""    
## [2,] "Trump" ""    
## [3,] "Karl"  "Marx"
\end{verbatim}

As the first two observations did not contain any white space, there was nothing to split there, and the function simply returned empty strings \texttt{""}. In a second step, we replace empty observations in the \texttt{first\_name} column with the corresponding values in \texttt{splitnames}.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{problem\_cases }\OtherTok{\textless{}{-}}\NormalTok{ messy\_df}\SpecialCharTok{$}\NormalTok{first\_name }\SpecialCharTok{==} \StringTok{""}
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{first\_name[problem\_cases] }\OtherTok{\textless{}{-}}\NormalTok{ splitnames[problem\_cases, }\DecValTok{1}\NormalTok{]}
\end{Highlighting}
\end{Shaded}

Finally, we must correct the \texttt{last\_name} column by replacing the respective values.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{last\_name[problem\_cases] }\OtherTok{\textless{}{-}}\NormalTok{ splitnames[problem\_cases, }\DecValTok{2}\NormalTok{]}
\NormalTok{messy\_df}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   last_name first_name gender       date income
## 1     Wayne       John   male 2018-11-15 150000
## 2     Trump    Melania female 2018.11.01 250000
## 3      Marx       Karl   male 2018/11/02  10000
\end{verbatim}

\hypertarget{parsing-dates}{%
\subsection{Parsing dates}\label{parsing-dates}}

Finally, we take a look at the \texttt{date}-column of our dataset. For many data preparation steps as well as visualization and analysis, it is advantageous to have times and dates properly parsed as type \texttt{Date}. In practice, dates and times are often particularly messy because no unique standard has been used to define the format in the data collection phase. This also seems to be the case in our dataset. In order to work with dates, we load the \texttt{lubridate} package.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{library}\NormalTok{(lubridate)}
\end{Highlighting}
\end{Shaded}

This package provides several functions to parse and manipulate date and time data. From the' date' column, we see that the format is year, month, and day. Thus, we can use the \texttt{ymd()}-function provided in the \texttt{lubridate}-package to parse the column as \texttt{Date} type.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{messy\_df}\SpecialCharTok{$}\NormalTok{date }\OtherTok{\textless{}{-}} \FunctionTok{ymd}\NormalTok{(messy\_df}\SpecialCharTok{$}\NormalTok{date)}
\end{Highlighting}
\end{Shaded}

Note how this function automatically recognizes how different special characters have been used in different observations to separate years from months/days.

Now, our dataset is cleaned up and ready to go.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{messy\_df}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   last_name first_name gender       date income
## 1     Wayne       John   male 2018-11-15 150000
## 2     Trump    Melania female 2018-11-01 250000
## 3      Marx       Karl   male 2018-11-02  10000
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{str}\NormalTok{(messy\_df)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 'data.frame':    3 obs. of  5 variables:
##  $ last_name : chr  "Wayne" "Trump" "Marx"
##  $ first_name: chr  "John" "Melania" "Karl"
##  $ gender    : Factor w/ 2 levels "female","male": 2 1 2
##  $ date      : Date, format: "2018-11-15" ...
##  $ income    : int  150000 250000 10000
\end{verbatim}

\hypertarget{reshaping-datasets}{%
\section{Reshaping datasets}\label{reshaping-datasets}}

Besides cleaning and standardizing individual data columns, preparing a dataset for analysis often involves bringing the entire dataset in the right `shape.' Typically, we mean this in a table-like (two-dimensional) format such as \texttt{data.\ frames} and \texttt{tibbles}, data with repeated observations for the same unit can be displayed/stored in either \emph{long} or \emph{wide} format. It is often seen as good practice to prepare data for analysis in \emph{long} (`tidy') format. This way we ensure that we follow the (`tidy') paradigm of using the rows for individual observations and the columns to describe these observations.\footnote{Depending on the dataset, however, an argument can be made that storing the data in wide format might be more efficient (using up less memory) than long format.} Tidying/reshaping a dataset in this way thus involves transforming columns into rows (i.e., \emph{melting} the dataset). In the following, we first have a close look at what this means conceptually and then apply this technique in two examples.

\hypertarget{tidying-messy-datasets.}{%
\subsection{Tidying messy datasets.}\label{tidying-messy-datasets.}}

Consider the following stylized example \citep{wickham_2014}.

\begin{tabular}{l|r|r}
\hline
person & treatmenta & treatmentb\\
\hline
John Smith & NA & 2\\
\hline
Jane Doe & 16 & 11\\
\hline
Mary Johnson & 3 & 1\\
\hline
\end{tabular}

The table shows observations of three individuals participating in an experiment. In this experiment, the subjects might have been exposed to treatment a and/or treatment b. Their reaction to either treatment is measured in numeric values (the results of the experiment). From looking at the raw data in its current shape, this is not really clear. While we see which numeric value corresponds to which person and treatment, it is unclear what this value is. One might, for example, wrongly assume that the numeric values refer to the treatment intensity of a and b. Such interpretation would align with the idea of columns containing variables and rows of observations. But, considering what the numeric values stand for, we realize that the columns are not \emph{names of variables} but \emph{values} of a variable (the categorical variable \texttt{treatment}, with levels \texttt{a} and \texttt{b}).

Now consider the same data in `tidy' format (variables in columns and observations in rows).

\begin{tabular}{l|l|r}
\hline
person & treatment & result\\
\hline
John Smith & a & NA\\
\hline
John Smith & b & 2\\
\hline
Jane Doe & a & 16\\
\hline
Jane Doe & b & 11\\
\hline
Mary Johnson & a & 3\\
\hline
Mary Johnson & b & 1\\
\hline
\end{tabular}

This \emph{long}/\emph{tidy} shape of the dataset has several advantages. First, it is now clear what the numeric values refer to. Second, it is much easier to filter/select the observations in this format.

\hypertarget{pivoting-from-wide-to-long}{%
\subsection{Pivoting from `wide to long'}\label{pivoting-from-wide-to-long}}

In the \texttt{tidyverse} context, we call the transformation of columns to rows `pivoting from wide to long'. That is, we pivot columns into keys (or names) and values. A typical situation where this has to be done in applied data analysis is when a dataset contains several observations over time for the same subjects. The following figure illustrates the basic concept. On the left, you see a wide data frame, which is not in line with the tidy data concept. Reshaping it to long format yields the new data frame on the right.

\begin{figure}

{\centering \includegraphics[width=0.4\linewidth]{img/reshape} 

}

\caption{Reshaping of a data frame from wide to long format.}\label{fig:widetolong}
\end{figure}



To illustrate how \emph{pivoting from wide to long} works in practice, consider the following example dataset (extending on the example above).

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{wide\_df }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{last\_name =} \FunctionTok{c}\NormalTok{(}\StringTok{"Wayne"}\NormalTok{, }\StringTok{"Trump"}\NormalTok{, }\StringTok{"Marx"}\NormalTok{),}
                       \AttributeTok{first\_name =} \FunctionTok{c}\NormalTok{(}\StringTok{"John"}\NormalTok{, }\StringTok{"Melania"}\NormalTok{, }\StringTok{"Karl"}\NormalTok{),}
                       \AttributeTok{gender =} \FunctionTok{c}\NormalTok{(}\StringTok{"male"}\NormalTok{, }\StringTok{"female"}\NormalTok{, }\StringTok{"male"}\NormalTok{),}
                       \AttributeTok{income.2018 =} \FunctionTok{c}\NormalTok{(}\StringTok{"150000"}\NormalTok{, }\StringTok{"250000"}\NormalTok{, }\StringTok{"10000"}\NormalTok{),}
                      \AttributeTok{income.2017 =} \FunctionTok{c}\NormalTok{( }\StringTok{"140000"}\NormalTok{, }\StringTok{"230000"}\NormalTok{, }\StringTok{"15000"}\NormalTok{),}
                      \AttributeTok{stringsAsFactors =} \ConstantTok{FALSE}\NormalTok{)}
\NormalTok{wide\_df}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   last_name first_name gender income.2018 income.2017
## 1     Wayne       John   male      150000      140000
## 2     Trump    Melania female      250000      230000
## 3      Marx       Karl   male       10000       15000
\end{verbatim}

The last two columns contain information on the same variable (\texttt{income}), but for different years. We thus want to pivot these two columns into a new \texttt{year} and \texttt{income} column, ensuring that columns correspond to variables and rows correspond to observations. For this, we call the \texttt{pivot\_longer()}-function as follows:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{long\_df }\OtherTok{\textless{}{-}} \FunctionTok{pivot\_longer}\NormalTok{(wide\_df, }\FunctionTok{c}\NormalTok{(income}\FloatTok{.2018}\NormalTok{, income}\FloatTok{.2017}\NormalTok{), }\AttributeTok{names\_to =} \StringTok{"year"}\NormalTok{, }\AttributeTok{values\_to =} \StringTok{"income"}\NormalTok{)}
\NormalTok{long\_df}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 6 x 5
##   last_name first_name gender year        income
##   <chr>     <chr>      <chr>  <chr>       <chr> 
## 1 Wayne     John       male   income.2018 150000
## 2 Wayne     John       male   income.2017 140000
## 3 Trump     Melania    female income.2018 250000
## 4 Trump     Melania    female income.2017 230000
## 5 Marx      Karl       male   income.2018 10000 
## 6 Marx      Karl       male   income.2017 15000
\end{verbatim}

We can further clean the \texttt{year} column to only contain the respective numeric values.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{long\_df}\SpecialCharTok{$}\NormalTok{year }\OtherTok{\textless{}{-}} \FunctionTok{str\_replace}\NormalTok{(long\_df}\SpecialCharTok{$}\NormalTok{year, }\StringTok{"income."}\NormalTok{, }\StringTok{""}\NormalTok{)}
\NormalTok{long\_df}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 6 x 5
##   last_name first_name gender year  income
##   <chr>     <chr>      <chr>  <chr> <chr> 
## 1 Wayne     John       male   2018  150000
## 2 Wayne     John       male   2017  140000
## 3 Trump     Melania    female 2018  250000
## 4 Trump     Melania    female 2017  230000
## 5 Marx      Karl       male   2018  10000 
## 6 Marx      Karl       male   2017  15000
\end{verbatim}

\hypertarget{pivoting-from-long-to-wide-spreading}{%
\subsection{Pivoting from `long to wide' (``spreading'')}\label{pivoting-from-long-to-wide-spreading}}

As we want to adhere to the `tidy' paradigm of keeping our data in long format, transforming `long to wide' is less common. However, it might be necessary if the dataset at hand is particularly messy. The following example illustrates such a situation.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{weird\_df }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{last\_name =} \FunctionTok{c}\NormalTok{(}\StringTok{"Wayne"}\NormalTok{, }\StringTok{"Trump"}\NormalTok{, }\StringTok{"Marx"}\NormalTok{,}
                                     \StringTok{"Wayne"}\NormalTok{, }\StringTok{"Trump"}\NormalTok{, }\StringTok{"Marx"}\NormalTok{,}
                                     \StringTok{"Wayne"}\NormalTok{, }\StringTok{"Trump"}\NormalTok{, }\StringTok{"Marx"}\NormalTok{),}
                       \AttributeTok{first\_name =} \FunctionTok{c}\NormalTok{(}\StringTok{"John"}\NormalTok{, }\StringTok{"Melania"}\NormalTok{, }\StringTok{"Karl"}\NormalTok{,}
                                      \StringTok{"John"}\NormalTok{, }\StringTok{"Melania"}\NormalTok{, }\StringTok{"Karl"}\NormalTok{,}
                                      \StringTok{"John"}\NormalTok{, }\StringTok{"Melania"}\NormalTok{, }\StringTok{"Karl"}\NormalTok{),}
                       \AttributeTok{gender =} \FunctionTok{c}\NormalTok{(}\StringTok{"male"}\NormalTok{, }\StringTok{"female"}\NormalTok{, }\StringTok{"male"}\NormalTok{,}
                                  \StringTok{"male"}\NormalTok{, }\StringTok{"female"}\NormalTok{, }\StringTok{"male"}\NormalTok{,}
                                  \StringTok{"male"}\NormalTok{, }\StringTok{"female"}\NormalTok{, }\StringTok{"male"}\NormalTok{),}
                       \AttributeTok{value =} \FunctionTok{c}\NormalTok{(}\StringTok{"150000"}\NormalTok{, }\StringTok{"250000"}\NormalTok{, }\StringTok{"10000"}\NormalTok{,}
                                 \StringTok{"2000000"}\NormalTok{, }\StringTok{"5000000"}\NormalTok{, }\StringTok{"NA"}\NormalTok{,}
                                 \StringTok{"50"}\NormalTok{, }\StringTok{"25"}\NormalTok{, }\StringTok{"NA"}\NormalTok{),}
                       \AttributeTok{variable =} \FunctionTok{c}\NormalTok{(}\StringTok{"income"}\NormalTok{, }\StringTok{"income"}\NormalTok{, }\StringTok{"income"}\NormalTok{,}
                                    \StringTok{"assets"}\NormalTok{, }\StringTok{"assets"}\NormalTok{, }\StringTok{"assets"}\NormalTok{,}
                                    \StringTok{"age"}\NormalTok{, }\StringTok{"age"}\NormalTok{, }\StringTok{"age"}\NormalTok{),}
                       \AttributeTok{stringsAsFactors =} \ConstantTok{FALSE}\NormalTok{)}
\NormalTok{weird\_df}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   last_name first_name gender   value variable
## 1     Wayne       John   male  150000   income
## 2     Trump    Melania female  250000   income
## 3      Marx       Karl   male   10000   income
## 4     Wayne       John   male 2000000   assets
## 5     Trump    Melania female 5000000   assets
## 6      Marx       Karl   male      NA   assets
## 7     Wayne       John   male      50      age
## 8     Trump    Melania female      25      age
## 9      Marx       Karl   male      NA      age
\end{verbatim}

While the data is somehow in a long format, the rule that each column should correspond to a variable (and vice versa) is ignored. Data on income, assets, and the age of the individuals in the dataset, are all put in the same column. We can call the function \texttt{pivot\_wider()} with the two parameters \texttt{names} and \texttt{value} to correct this.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{tidy\_df }\OtherTok{\textless{}{-}} \FunctionTok{pivot\_wider}\NormalTok{(weird\_df, }\AttributeTok{names\_from =} \StringTok{"variable"}\NormalTok{, }\AttributeTok{values\_from =} \StringTok{"value"}\NormalTok{)}
\NormalTok{tidy\_df}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 3 x 6
##   last_name first_name gender income assets  age  
##   <chr>     <chr>      <chr>  <chr>  <chr>   <chr>
## 1 Wayne     John       male   150000 2000000 50   
## 2 Trump     Melania    female 250000 5000000 25   
## 3 Marx      Karl       male   10000  NA      NA
\end{verbatim}

\hypertarget{tutorial-hotel-bookings-time-series}{%
\section{Tutorial: Hotel Bookings Time Series}\label{tutorial-hotel-bookings-time-series}}

This tutorial guides you step-by-step through the cleaning script (with a few adaptions) of \href{https://github.com/rfordatascience/tidytuesday/tree/master/data/2020/2020-02-11}{tidytuesday's Hotel Bookings repo}, dealing with the preparation and analysis of two datasets with \emph{hotel demand data}. Along the way, you also get in touch with the \href{https://github.com/sfirke/janitor}{janitor package}. For details about the two datasets, see the \href{https://www.sciencedirect.com/science/article/pii/S2352340918315191\#f0010}{paper} by \citet{nuno_etal2019}, and for the original research contribution related to these datasets see the \href{https://ieeexplore.ieee.org/document/8260781}{paper} by \citet{nunes_etal2017}.

\citet{nuno_etal2019} summarizes the content of the datasets as follows: ``One of the hotels (H1) is a resort hotel, and the other is a city hotel (H2). Both datasets share the same structure, with 31 variables describing the 40,060 observations of H1 and 79,330 observations of H2. Each observation represents a hotel booking. Both datasets comprehend bookings due to arrive between the 1st of July 2015 and the 31st of August 2017, including bookings that effectively arrived and bookings that were canceled. Since this is real data, all data elements pertaining to hotel or customer identification were deleted. Due to the scarcity of real business data for scientific and educational purposes, these datasets can have an important role for research and education in revenue management, machine learning, or data mining, as well as in other fields.''

The aim of the tutorial is to get the data in the form needed for the following plot.

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-144-1.pdf}

The first few rows and columns of the final dataset should combine the two source datasets and look as follows:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{head}\NormalTok{(hotel\_df)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 6 x 32
##   hotel   is_canceled lead_time arrival_date_ye~ arrival_date_mo~
##   <chr>         <dbl>     <dbl>            <dbl> <chr>           
## 1 Resort~           0       342             2015 July            
## 2 Resort~           0       737             2015 July            
## 3 Resort~           0         7             2015 July            
## 4 Resort~           0        13             2015 July            
## 5 Resort~           0        14             2015 July            
## 6 Resort~           0        14             2015 July            
## # ... with 27 more variables: arrival_date_week_number <dbl>,
## #   arrival_date_day_of_month <dbl>,
## #   stays_in_weekend_nights <dbl>, stays_in_week_nights <dbl>,
## #   adults <dbl>, children <dbl>, babies <dbl>, meal <chr>,
## #   country <chr>, market_segment <chr>,
## #   distribution_channel <chr>, is_repeated_guest <dbl>,
## #   previous_cancellations <dbl>, ...
\end{verbatim}

\hypertarget{set-up-and-import}{%
\section{Set up and import}\label{set-up-and-import}}

All the tools we need for this tutorial are provided in \texttt{tidyverse} and \texttt{janitor}, and the data is directly available from the \href{https://github.com/rfordatascience/tidytuesday/tree/master/data/2020/2020-02-11}{tidytuesday GitHub repository}. The original data is provided in CSV format.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# SET UP {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(tidyverse)}
\FunctionTok{library}\NormalTok{(janitor) }\CommentTok{\# install.packages("janitor") (if not yet installed)}

\CommentTok{\# fix variables}
\NormalTok{url\_h1 }\OtherTok{\textless{}{-}} \StringTok{"https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020{-}02{-}11/H1.csv"}
\NormalTok{url\_h2 }\OtherTok{\textless{}{-}} \StringTok{"https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020{-}02{-}11/H2.csv"}

\DocumentationTok{\#\# DATA IMPORT {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\NormalTok{h1 }\OtherTok{\textless{}{-}} \FunctionTok{read\_csv}\NormalTok{(url\_h1)}
\NormalTok{h2 }\OtherTok{\textless{}{-}} \FunctionTok{read\_csv}\NormalTok{(url\_h2)}
\end{Highlighting}
\end{Shaded}

In the next step, we clean the column names and add columns to clarify which of the two hotels the corresponding observations belong to (see dataset description above). Finally, we stack the observations (rows) together in one tibble/data.frame.

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# CLEAN DATA {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\CommentTok{\# use the janitor{-}package clean\_names function. see ?clean\_names for details}
\NormalTok{h1 }\OtherTok{\textless{}{-}} \FunctionTok{clean\_names}\NormalTok{(h1)}
\NormalTok{h2 }\OtherTok{\textless{}{-}} \FunctionTok{clean\_names}\NormalTok{(h2)}

\CommentTok{\# add a column to clarify the origin of observation}
\NormalTok{h1 }\OtherTok{\textless{}{-}} \FunctionTok{mutate}\NormalTok{(h1, }\AttributeTok{hotel=}\StringTok{"Resort Hotel"}\NormalTok{)}
\NormalTok{h2 }\OtherTok{\textless{}{-}} \FunctionTok{mutate}\NormalTok{(h2, }\AttributeTok{hotel=}\StringTok{"City Hotel"}\NormalTok{)}

\CommentTok{\# stack observations}
\NormalTok{hotel\_df }\OtherTok{\textless{}{-}} \FunctionTok{bind\_rows}\NormalTok{(h1,h2)}

\CommentTok{\# inspect the first observations}
\FunctionTok{head}\NormalTok{(hotel\_df)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 6 x 32
##   is_canceled lead_time arrival_date_year arrival_date_month
##         <dbl>     <dbl>             <dbl> <chr>             
## 1           0       342              2015 July              
## 2           0       737              2015 July              
## 3           0         7              2015 July              
## 4           0        13              2015 July              
## 5           0        14              2015 July              
## 6           0        14              2015 July              
## # ... with 28 more variables: arrival_date_week_number <dbl>,
## #   arrival_date_day_of_month <dbl>,
## #   stays_in_weekend_nights <dbl>, stays_in_week_nights <dbl>,
## #   adults <dbl>, children <dbl>, babies <dbl>, meal <chr>,
## #   country <chr>, market_segment <chr>,
## #   distribution_channel <chr>, is_repeated_guest <dbl>,
## #   previous_cancellations <dbl>, ...
\end{verbatim}

\hypertarget{data-analysis-first-steps}{%
\chapter{Data Analysis: First Steps}\label{data-analysis-first-steps}}

In the first part of this chapter, we look at some key functions for applied data analysis in R. At this point, we have already implemented collecting/importing and cleaning the raw data. The analysis part can be thought of as a collection of tasks with the aim of making sense of the data. In practice, this can be explorative (discovering interesting patterns in the data) or inductive (testing of a specific hypothesis). Moreover, it typically involves functions for actual statistical analysis and various functions to select, combine, filter, and aggregate data. Similar to the topic of data cleaning/preparation, covering all aspects of applied data analysis with R goes well beyond the scope of one chapter. The aim is thus to give a practical overview of some of the basic concepts and their corresponding R functions (here from \texttt{tidyverse}).

\hypertarget{mergingjoining-datasets}{%
\section{Merging/joining datasets}\label{mergingjoining-datasets}}

The following two data sets contain data on persons' characteristics and their consumption spending. Both are cleaned datasets. But, for analysis purposes, we have to combine the two datasets. There are several ways to do this in R, but most commonly (for \texttt{data.frames} as well as \texttt{tibbles}), we can use the \texttt{merge()}-function.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(tidyverse)}

\CommentTok{\# initiate data frame on persons\textquotesingle{} spending}
\NormalTok{df\_c }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{id =} \FunctionTok{c}\NormalTok{(}\DecValTok{1}\SpecialCharTok{:}\DecValTok{3}\NormalTok{,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{3}\NormalTok{),}
                   \AttributeTok{money\_spent=} \FunctionTok{c}\NormalTok{(}\DecValTok{1000}\NormalTok{, }\DecValTok{2000}\NormalTok{, }\DecValTok{6000}\NormalTok{, }\DecValTok{1500}\NormalTok{, }\DecValTok{3000}\NormalTok{, }\DecValTok{5500}\NormalTok{),}
                   \AttributeTok{currency =} \FunctionTok{c}\NormalTok{(}\StringTok{"CHF"}\NormalTok{, }\StringTok{"CHF"}\NormalTok{, }\StringTok{"USD"}\NormalTok{, }\StringTok{"EUR"}\NormalTok{, }\StringTok{"CHF"}\NormalTok{, }\StringTok{"USD"}\NormalTok{),}
                   \AttributeTok{year=}\FunctionTok{c}\NormalTok{(}\DecValTok{2017}\NormalTok{,}\DecValTok{2017}\NormalTok{,}\DecValTok{2017}\NormalTok{,}\DecValTok{2018}\NormalTok{,}\DecValTok{2018}\NormalTok{,}\DecValTok{2018}\NormalTok{))}
\NormalTok{df\_c}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   id money_spent currency year
## 1  1        1000      CHF 2017
## 2  2        2000      CHF 2017
## 3  3        6000      USD 2017
## 4  1        1500      EUR 2018
## 5  2        3000      CHF 2018
## 6  3        5500      USD 2018
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# initiate data frame on persons\textquotesingle{} characteristics}
\NormalTok{df\_p }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{id =} \DecValTok{1}\SpecialCharTok{:}\DecValTok{4}\NormalTok{,}
                   \AttributeTok{first\_name =} \FunctionTok{c}\NormalTok{(}\StringTok{"Anna"}\NormalTok{, }\StringTok{"Betty"}\NormalTok{, }\StringTok{"Claire"}\NormalTok{, }\StringTok{"Diane"}\NormalTok{),}
                   \AttributeTok{profession =} \FunctionTok{c}\NormalTok{(}\StringTok{"Economist"}\NormalTok{, }\StringTok{"Data Scientist"}\NormalTok{,}
                                  \StringTok{"Data Scientist"}\NormalTok{, }\StringTok{"Economist"}\NormalTok{))}
\NormalTok{df\_p}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   id first_name     profession
## 1  1       Anna      Economist
## 2  2      Betty Data Scientist
## 3  3     Claire Data Scientist
## 4  4      Diane      Economist
\end{verbatim}

Our aim is to compute the average spending by profession. Therefore, we want to link \texttt{money\_spent} with \texttt{profession}. Both datasets contain a unique identifier \texttt{id}, which we can use to link the observations via \texttt{merge()}.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{df\_merged }\OtherTok{\textless{}{-}} \FunctionTok{merge}\NormalTok{(df\_p, df\_c, }\AttributeTok{by=}\StringTok{"id"}\NormalTok{)}
\NormalTok{df\_merged}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   id first_name     profession money_spent currency year
## 1  1       Anna      Economist        1000      CHF 2017
## 2  1       Anna      Economist        1500      EUR 2018
## 3  2      Betty Data Scientist        2000      CHF 2017
## 4  2      Betty Data Scientist        3000      CHF 2018
## 5  3     Claire Data Scientist        6000      USD 2017
## 6  3     Claire Data Scientist        5500      USD 2018
\end{verbatim}

Note how only the exact matches are merged. The observation of \texttt{"Diane"} is not part of the merged data frame because there is no corresponding row in \texttt{df\_c} with her spending information. This approach to merging two data sets (only keeping the matched rows on both sides) is often referred to as an \emph{inner join}. If for some reason, we would like to have all persons in the merged dataset, we can specify the \texttt{merge()}-call accordingly:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{df\_merged2 }\OtherTok{\textless{}{-}} \FunctionTok{merge}\NormalTok{(df\_p, df\_c, }\AttributeTok{by=}\StringTok{"id"}\NormalTok{, }\AttributeTok{all =} \ConstantTok{TRUE}\NormalTok{)}
\NormalTok{df\_merged2}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   id first_name     profession money_spent currency year
## 1  1       Anna      Economist        1000      CHF 2017
## 2  1       Anna      Economist        1500      EUR 2018
## 3  2      Betty Data Scientist        2000      CHF 2017
## 4  2      Betty Data Scientist        3000      CHF 2018
## 5  3     Claire Data Scientist        6000      USD 2017
## 6  3     Claire Data Scientist        5500      USD 2018
## 7  4      Diane      Economist          NA     <NA>   NA
\end{verbatim}

this version of merging two data sets is also often referred to as an \emph{outer join}.

Finally, you might be in a situation in which keeping all rows of the first (left) dataset or the ones of the second (right) dataset makes sense, but not both. You can do this by specifying \texttt{all.x=TRUE} (keep all rows of the first dataset) or \texttt{all.y=TRUE} (keep all of the second dataset). These types of merging datasets are often also referred to \emph{left join} and \emph{right join}, respectively.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# left join}
\NormalTok{df\_merged3 }\OtherTok{\textless{}{-}} \FunctionTok{merge}\NormalTok{(df\_p, df\_c, }\AttributeTok{by=}\StringTok{"id"}\NormalTok{, }\AttributeTok{all.x =} \ConstantTok{TRUE}\NormalTok{)}
\NormalTok{df\_merged3}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   id first_name     profession money_spent currency year
## 1  1       Anna      Economist        1000      CHF 2017
## 2  1       Anna      Economist        1500      EUR 2018
## 3  2      Betty Data Scientist        2000      CHF 2017
## 4  2      Betty Data Scientist        3000      CHF 2018
## 5  3     Claire Data Scientist        6000      USD 2017
## 6  3     Claire Data Scientist        5500      USD 2018
## 7  4      Diane      Economist          NA     <NA>   NA
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# right join}
\NormalTok{df\_merged4 }\OtherTok{\textless{}{-}} \FunctionTok{merge}\NormalTok{(df\_p, df\_c, }\AttributeTok{by=}\StringTok{"id"}\NormalTok{, }\AttributeTok{all.y =} \ConstantTok{TRUE}\NormalTok{)}
\NormalTok{df\_merged4}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   id first_name     profession money_spent currency year
## 1  1       Anna      Economist        1000      CHF 2017
## 2  1       Anna      Economist        1500      EUR 2018
## 3  2      Betty Data Scientist        2000      CHF 2017
## 4  2      Betty Data Scientist        3000      CHF 2018
## 5  3     Claire Data Scientist        6000      USD 2017
## 6  3     Claire Data Scientist        5500      USD 2018
\end{verbatim}

The following figure illustrates the four types of joins.

\begin{figure}

{\centering \includegraphics[width=0.99\linewidth]{img/joins} 

}

\caption{Different approaches to merging two datasets: left join, right join, inner join, outer join.}\label{fig:joins}
\end{figure}



\hypertarget{selecting-subsets}{%
\section{Selecting subsets}\label{selecting-subsets}}

For our analysis's next steps, we do not need to have all columns. Via the \texttt{select()}-function provided in \texttt{tidyverse} we can easily select the columns of interest

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{df\_selection }\OtherTok{\textless{}{-}} \FunctionTok{select}\NormalTok{(df\_merged, id, year, money\_spent, currency)}
\NormalTok{df\_selection}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   id year money_spent currency
## 1  1 2017        1000      CHF
## 2  1 2018        1500      EUR
## 3  2 2017        2000      CHF
## 4  2 2018        3000      CHF
## 5  3 2017        6000      USD
## 6  3 2018        5500      USD
\end{verbatim}

\hypertarget{filtering-datasets}{%
\subsection{Filtering datasets}\label{filtering-datasets}}

In the next step, we want to select only observations with specific characteristics. Say we want to select only observations from 2018. Again there are several ways to do this in R, but the most comfortable way is to use the \texttt{filter()} function provided in \texttt{tidyverse}.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{filter}\NormalTok{(df\_selection, year }\SpecialCharTok{==} \DecValTok{2018}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   id year money_spent currency
## 1  1 2018        1500      EUR
## 2  2 2018        3000      CHF
## 3  3 2018        5500      USD
\end{verbatim}

We can use several filtering conditions simultaneously:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{filter}\NormalTok{(df\_selection, year }\SpecialCharTok{==} \DecValTok{2018}\NormalTok{, money\_spent }\SpecialCharTok{\textless{}} \DecValTok{5000}\NormalTok{, currency}\SpecialCharTok{==}\StringTok{"EUR"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   id year money_spent currency
## 1  1 2018        1500      EUR
\end{verbatim}

\hypertarget{mutating-datasets}{%
\subsection{Mutating datasets}\label{mutating-datasets}}

Before we compute aggregate statistics based on our selected dataset, we have to deal with the fact that the \texttt{money\_spent}-variable is not tidy. It describes each observation's characteristic, but it is measured in different units (here, different currencies) across some of these observations. If the aim was to have a perfectly tidy dataset, we could address the issue with \texttt{spread()}. However, in this context, it could be more helpful to add an additional variable/column with a normalized amount of money spent. That is, we want to have every value converted to one currency (given a certain exchange rate). In order to do so, we use the \texttt{mutate()} function (again provided in \texttt{tidyverse}).

First, we look up the USD/CHF and EUR/CHF exchange rates and add those as a variable (CHF/CHF exchange rates are equal to 1, of course).

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{exchange\_rates }\OtherTok{\textless{}{-}} \FunctionTok{data.frame}\NormalTok{(}\AttributeTok{exchange\_rate=} \FunctionTok{c}\NormalTok{(}\FloatTok{0.9}\NormalTok{, }\DecValTok{1}\NormalTok{, }\FloatTok{1.2}\NormalTok{),}
                             \AttributeTok{currency=}\FunctionTok{c}\NormalTok{(}\StringTok{"USD"}\NormalTok{, }\StringTok{"CHF"}\NormalTok{, }\StringTok{"EUR"}\NormalTok{), }\AttributeTok{stringsAsFactors =} \ConstantTok{FALSE}\NormalTok{)}
\NormalTok{df\_selection }\OtherTok{\textless{}{-}} \FunctionTok{merge}\NormalTok{(df\_selection, exchange\_rates, }\AttributeTok{by=}\StringTok{"currency"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

Now we can define an additional variable with the money spent in CHF via \texttt{mutate()}:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{df\_mutated }\OtherTok{\textless{}{-}} \FunctionTok{mutate}\NormalTok{(df\_selection, }\AttributeTok{money\_spent\_chf =}\NormalTok{ money\_spent }\SpecialCharTok{*}\NormalTok{ exchange\_rate)}
\NormalTok{df\_mutated}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   currency id year money_spent exchange_rate money_spent_chf
## 1      CHF  1 2017        1000           1.0            1000
## 2      CHF  2 2017        2000           1.0            2000
## 3      CHF  2 2018        3000           1.0            3000
## 4      EUR  1 2018        1500           1.2            1800
## 5      USD  3 2017        6000           0.9            5400
## 6      USD  3 2018        5500           0.9            4950
\end{verbatim}

\hypertarget{aggregation-and-summary-statistics}{%
\subsection{Aggregation and summary statistics}\label{aggregation-and-summary-statistics}}

Now we can start analyzing the dataset. Typically, the first step of analyzing a dataset is to get an overview by computing some summary statistics. This helps to better understand the dataset at hand. Key summary statistics of the variables of interest are the mean, standard deviation, median, and a number of observations. Together, they give a first idea of how the variables of interest are distributed.

As you know from previous chapters, R has several built-in functions that help us do this. In practice, these basic functions are often combined with functions implemented particularly for this step of the analysis, such as \texttt{summarise()} provided in \texttt{tidyverse}.

As the first output in our report, we want to show the key characteristics of the spending data in one table.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summarise}\NormalTok{(df\_mutated,}
          \AttributeTok{mean =} \FunctionTok{mean}\NormalTok{(money\_spent\_chf),}
          \AttributeTok{standard\_deviation =} \FunctionTok{sd}\NormalTok{(money\_spent\_chf),}
          \AttributeTok{median =} \FunctionTok{median}\NormalTok{(money\_spent\_chf),}
          \AttributeTok{N =} \FunctionTok{n}\NormalTok{())}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   mean standard_deviation median N
## 1 3025               1789   2500 6
\end{verbatim}

Moreover, we can compute the same statistics grouped by certain observation characteristics. For example, we can compute the same summary statistics per year of observation.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{by\_year }\OtherTok{\textless{}{-}} \FunctionTok{group\_by}\NormalTok{(df\_mutated, year)}
\FunctionTok{summarise}\NormalTok{(by\_year,}
          \AttributeTok{mean =} \FunctionTok{mean}\NormalTok{(money\_spent\_chf),}
          \AttributeTok{standard\_deviation =} \FunctionTok{sd}\NormalTok{(money\_spent\_chf),}
          \AttributeTok{median =} \FunctionTok{median}\NormalTok{(money\_spent\_chf),}
          \AttributeTok{N =} \FunctionTok{n}\NormalTok{())}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 2 x 5
##    year  mean standard_deviation median     N
##   <dbl> <dbl>              <dbl>  <dbl> <int>
## 1  2017  2800              2307.   2000     3
## 2  2018  3250              1590.   3000     3
\end{verbatim}

Alternatively, to the more user-friendly (but less flexible) \texttt{summarise} function, we can use lower-level functions to compute aggregate statistics provided in the basic R distribution. A good example of such a function is \texttt{sapply()}. In simple terms, \texttt{sapply()} takes a list as input and applies a function to the content of each element in this list (here: compute a statistic for each column). To illustrate this point, we load the already familiar \texttt{swiss} dataset.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load data}
\FunctionTok{data}\NormalTok{(}\StringTok{"swiss"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

Now we want to compute the mean for each variable in this dataset. Technically speaking, a data frame is a list, where each list element is a column of the same length. Thus, we can use \texttt{sapply()} to `apply' the function \texttt{mean()} to each of the columns in \texttt{swiss}.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{sapply}\NormalTok{(swiss, mean)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##        Fertility      Agriculture      Examination 
##            70.14            50.66            16.49 
##        Education         Catholic Infant.Mortality 
##            10.98            41.14            19.94
\end{verbatim}

By default, \texttt{sapply()} returns a vector or a matrix.\footnote{The related function \texttt{lapply()}, returns a list (see \texttt{lapply(swiss,\ mean)}).} We can get a similar result by using \texttt{summarise()}. However, we would have to explicitly mention which variables we want as input.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summarise}\NormalTok{(swiss,}
          \AttributeTok{Fertility =} \FunctionTok{mean}\NormalTok{(Fertility),}
          \AttributeTok{Agriculture =} \FunctionTok{mean}\NormalTok{(Agriculture)) }\CommentTok{\# etc.}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   Fertility Agriculture
## 1     70.14       50.66
\end{verbatim}

\hypertarget{tutorial-analise-messy-excel-sheets}{%
\section{Tutorial: Analise messy Excel sheets}\label{tutorial-analise-messy-excel-sheets}}

The following tutorial is a (substantially) shortened and simplified version of Ista Zahn and Daina Bouquin's \href{https://rawgit.com/izahn/R-data-cleaning/master/dataCleaning.html}{``Cleaning up messy data tutorial'' (Harvard Datafest 2017)}. The tutorial aims to clean up an Excel sheet provided by the UK Office of National Statistics that provides data on the most popular baby names in England and Wales in 2015. The dataset is stored in \texttt{data/2015boysnamesfinal.xlsx}

\hypertarget{preparatory-steps}{%
\subsection{Preparatory steps}\label{preparatory-steps}}

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# SET UP {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}
\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(tidyverse)}
\FunctionTok{library}\NormalTok{(readxl)}

\CommentTok{\# fix variables}
\NormalTok{INPUT\_PATH }\OtherTok{\textless{}{-}} \StringTok{"data/2015boysnamesfinal.xlsx"}
\end{Highlighting}
\end{Shaded}

Before diving into the data import and cleaning, it is helpful to first open the file in Excel. We notice a couple of things there: first, there are several sheets in this Excel file. For this exercise, we only rely on the sheet called ``Table 1''. Second, in this sheet, we notice intuitively some potential problems with importing this dataset due to the way the spreadsheet is organized. The actual data entries only start on row 7. These two issues can be considered when importing the data with \texttt{read\_excel()}.

\begin{Shaded}
\begin{Highlighting}[]
\DocumentationTok{\#\# LOAD/INSPECT DATA {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\CommentTok{\# import the excel sheet}
\NormalTok{boys }\OtherTok{\textless{}{-}} \FunctionTok{read\_excel}\NormalTok{(INPUT\_PATH, }\AttributeTok{col\_names =} \ConstantTok{TRUE}\NormalTok{,}
                   \AttributeTok{sheet =} \StringTok{"Table 1"}\NormalTok{, }\CommentTok{\# the name of the sheet to be loaded into R}
                   \AttributeTok{skip =} \DecValTok{6} \CommentTok{\# skip the first 6 rows of the original sheet,}
\NormalTok{                   )}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## New names:
## * `Rank` -> `Rank...1`
## * `Name` -> `Name...2`
## * `Count` -> `Count...3`
## * `since 2014` -> `since 2014...4`
## * `since 2005` -> `since 2005...5`
## * `` -> `...6`
## * `Rank` -> `Rank...7`
## * `Name` -> `Name...8`
## * `Count` -> `Count...9`
## * `since 2014` -> `since 2014...10`
## * `since 2005` -> `since 2005...11`
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# inspect}
\NormalTok{boys}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 61 x 11
##    Rank...1 Name...2 Count...3 `since 2014...4` `since 2005...5`
##    <chr>    <chr>        <dbl> <chr>            <chr>           
##  1 <NA>     <NA>            NA <NA>             <NA>            
##  2 1        OLIVER        6941 ­                 +4              
##  3 2        JACK          5371 ­                 -1              
##  4 3        HARRY         5308 ­                 +6              
##  5 4        GEORGE        4869 +3               +13             
##  6 5        JACOB         4850 -1               +16             
##  7 6        CHARLIE       4831 -1               +6              
##  8 7        NOAH          4148 +4               +44             
##  9 8        WILLIAM       4083 +2               ­                
## 10 9        THOMAS        4075 -3               -6              
## # ... with 51 more rows, and 6 more variables: ...6 <lgl>,
## #   Rank...7 <chr>, Name...8 <chr>, Count...9 <dbl>,
## #   `since 2014...10` <chr>, `since 2005...11` <chr>
\end{verbatim}

Note that by default, \texttt{read\_excel()} ``repairs'' the column names of imported datasets to ensure all columns have unique names. We do not need to worry about the automatically assigned column names. However, some of the columns are not needed for analytics purposes. In addition, we note that some rows are empty (contain \texttt{NA} values). In the next step we \emph{select} only those columns needed and \emph{filter} incomplete observations out.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# FILTER/CLEAN {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\CommentTok{\# select columns}
\NormalTok{boys }\OtherTok{\textless{}{-}} \FunctionTok{select}\NormalTok{(boys, Rank...}\DecValTok{1}\NormalTok{, Name...}\DecValTok{2}\NormalTok{, Count...}\DecValTok{3}\NormalTok{, Rank...}\DecValTok{7}\NormalTok{, Name...}\DecValTok{8}\NormalTok{, Count...}\DecValTok{9}\NormalTok{)}
\CommentTok{\# filter rows}
\NormalTok{boys }\OtherTok{\textless{}{-}}  \FunctionTok{filter}\NormalTok{(boys, }\SpecialCharTok{!}\FunctionTok{is.na}\NormalTok{(Rank...}\DecValTok{1}\NormalTok{))}
\end{Highlighting}
\end{Shaded}

Finally, we re-arrange the data by stacking them in a three-column format.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# stack columns}
\NormalTok{boys\_long }\OtherTok{\textless{}{-}} \FunctionTok{bind\_rows}\NormalTok{(boys[,}\DecValTok{1}\SpecialCharTok{:}\DecValTok{3}\NormalTok{], boys[,}\DecValTok{4}\SpecialCharTok{:}\DecValTok{6}\NormalTok{])}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## New names:
## New names:
## * `Rank...1` -> `Rank`
## * `Name...2` -> `Name`
## * `Count...3` -> `Count`
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# inspect result}
\NormalTok{boys\_long}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 114 x 3
##    Rank  Name    Count
##    <chr> <chr>   <dbl>
##  1 1     OLIVER   6941
##  2 2     JACK     5371
##  3 3     HARRY    5308
##  4 4     GEORGE   4869
##  5 5     JACOB    4850
##  6 6     CHARLIE  4831
##  7 7     NOAH     4148
##  8 8     WILLIAM  4083
##  9 9     THOMAS   4075
## 10 10    OSCAR    4066
## # ... with 104 more rows
\end{verbatim}

\hypertarget{basic-econometrics-in-r}{%
\chapter{Basic Econometrics in R}\label{basic-econometrics-in-r}}

When we look at practical data analytics in an economics context, it becomes quickly apparent that the vast majority of applied econometric research builds in one way or the other on linear regression, specifically on \emph{ordinary least squares (OLS)}. OLS is the bread-and-butter model in microeconometrics for several reasons: it is rather easy to understand, its results are straightforward to interpret, it comes with several very useful statistical properties, and it can easily be applied to a large variety of economic research questions. In this section, we get to know the very basic idea behind OLS by looking at it through the lens of formal math notation as well as through the lens of R code.

\hypertarget{statistical-modeling}{%
\section{Statistical modeling}\label{statistical-modeling}}

Before we turn to a specific data example, let us introduce a general notation for this econometric problem. Generally we denote the \emph{dependent variable} as \(y_i\) and the \emph{explanatory variable} as \(x_i\). All the rest that is not more specifically explained by explanatory variables (\(u_{i}\)) we call the \emph{' residuals'} or the `error term.' Hence we can rewrite the problem with a more general notation as

\(y_{i}= \alpha + \beta x_{i} + u_{i}\).

The overarching goal of modern econometrics for such problems is to assess whether \(x\) has a \emph{causal} effect on \(y\). A key aspect of such an interpretation is that \(u_{i}\) is unrelated to \(x_i\). Other unobservable factors might also play a role for \(y\), but they must not affect both \(y\) and \(x\). The great computer scientist \href{https://en.wikipedia.org/wiki/Judea_Pearl}{Judea Pearl} has introduced an intuitive approach to illustrating such causality problems. In the left panel of the figure below, we illustrate the unfortunate case where we have a so-called \emph{' endogeneity'} problem, a situation in which other factors affect both \(x\) and \(y\).

\begin{figure}

{\centering \includegraphics[width=0.6\linewidth]{img/causal_diagram} 

}

\caption{Causal diagrams. On the left with an endogeneity issue (other factors affect both the dependent and the explanatory variable), and on the right without this endogeneity issue.}\label{fig:causality}
\end{figure}



\hypertarget{illustration-with-pseudo-data}{%
\subsection{Illustration with pseudo-data}\label{illustration-with-pseudo-data}}

Let us now look at how we can estimate \(\beta\) under the assumption that \(u\) does not affect \(x\). To make this more general and easily accessible, let us, for the moment, ignore the dataset above and generate simulated data for which this assumption certainly holds. The basic R installation provides a number of functions to easily generate vectors of pseudo-random numbers\footnote{Computers cannot actually `generate' true random numbers. However, there are functions that produce series of numbers which have the properties of randomly drawn numbers from a given distribution.} The great advantage of using simulated data and code to understand statistical models is that we have full control over what the \emph{actual} relationship between variables is. We can then easily assess how well an estimation procedure for a parameter is by comparing the parameter estimate with its true value (something we never observe when working with real-life data).

First, we define the key parameters for the simulation. We choose the actual values of \(\alpha\) and \(\beta\), and set the number of observations \(N\).

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{alpha }\OtherTok{\textless{}{-}} \DecValTok{30}
\NormalTok{beta }\OtherTok{\textless{}{-}} \FloatTok{0.9}
\NormalTok{N }\OtherTok{\textless{}{-}} \DecValTok{1000}
\end{Highlighting}
\end{Shaded}

Now, we initiate a vector \(x\) of length \(N\) drawn from the uniform distribution (between 0 and 0.5). This will be our explanatory variable.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{x }\OtherTok{\textless{}{-}} \FunctionTok{runif}\NormalTok{(N, }\DecValTok{0}\NormalTok{, }\DecValTok{50}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

Next, we draw a vector of random errors (residuals) \(u\) (from a normal distribution with mean=0 and SD=0.05) and compute the corresponding values of the dependent variable \(y\). Since we impose that the actual relationship between the variables is exactly defined in our simple linear model, this computation is straightforward (\(y_{i}= \alpha + \beta x_{i} + u_{i}\)).

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# draw the random errors (all the other factors also affecting y)}
\NormalTok{epsilon }\OtherTok{\textless{}{-}} \FunctionTok{rnorm}\NormalTok{(N, }\AttributeTok{sd=}\DecValTok{10}\NormalTok{)}
\CommentTok{\# compute the dependent variable values}
\NormalTok{y }\OtherTok{\textless{}{-}}\NormalTok{ alpha }\SpecialCharTok{+}\NormalTok{ beta}\SpecialCharTok{*}\NormalTok{x }\SpecialCharTok{+}\NormalTok{ epsilon}
\end{Highlighting}
\end{Shaded}

We can illustrate how \(y\) changes when \(x\) increases by plotting the raw data as a scatter plot as well as the true relationship (function) as a line.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{plot}\NormalTok{(x,y)}
\FunctionTok{abline}\NormalTok{(}\AttributeTok{a =}\NormalTok{ alpha, }\AttributeTok{b=}\NormalTok{beta, }\AttributeTok{col=}\StringTok{"red"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-171-1.pdf}

We see that while individual observations can be way off the true values (the red line), `on average' the observations seem to follow the true relationship between \(x\) and \(y\). We can look at this closely by computing the average observed \(y\)-values for different intervals along the \(x\)-axis.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# compute average y per x intervals}
\NormalTok{lower }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\DecValTok{0}\NormalTok{,}\DecValTok{10}\NormalTok{,}\DecValTok{20}\NormalTok{,}\DecValTok{30}\NormalTok{,}\DecValTok{40}\NormalTok{)}
\NormalTok{upper }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(lower[}\SpecialCharTok{{-}}\DecValTok{1}\NormalTok{], }\DecValTok{50}\NormalTok{)}
\NormalTok{n\_intervals }\OtherTok{\textless{}{-}} \FunctionTok{length}\NormalTok{(lower)}
\NormalTok{y\_bars }\OtherTok{\textless{}{-}} \FunctionTok{list}\NormalTok{()}
\FunctionTok{length}\NormalTok{(y\_bars) }\OtherTok{\textless{}{-}}\NormalTok{ n\_intervals}
\NormalTok{x\_bars }\OtherTok{\textless{}{-}} \FunctionTok{list}\NormalTok{()}
\FunctionTok{length}\NormalTok{(x\_bars) }\OtherTok{\textless{}{-}}\NormalTok{ n\_intervals}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in} \DecValTok{1}\SpecialCharTok{:}\NormalTok{n\_intervals)\{}
\NormalTok{  y\_bars[i] }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(y[lower[i] }\SpecialCharTok{\textless{}=}\NormalTok{ x }\SpecialCharTok{\&}\NormalTok{ x }\SpecialCharTok{\textless{}}\NormalTok{ upper[i]])}
\NormalTok{  x\_bars[i] }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(x[lower[i] }\SpecialCharTok{\textless{}=}\NormalTok{ x }\SpecialCharTok{\&}\NormalTok{ x }\SpecialCharTok{\textless{}}\NormalTok{ upper[i]])}

\NormalTok{\}}
\NormalTok{y\_bars }\OtherTok{\textless{}{-}} \FunctionTok{unlist}\NormalTok{(y\_bars)}
\NormalTok{x\_bars }\OtherTok{\textless{}{-}} \FunctionTok{unlist}\NormalTok{(x\_bars)}

\CommentTok{\# add to plot}
\FunctionTok{plot}\NormalTok{(x,y)}
\FunctionTok{abline}\NormalTok{(}\AttributeTok{a =}\NormalTok{ alpha, }\AttributeTok{b=}\NormalTok{beta, }\AttributeTok{col=}\StringTok{"red"}\NormalTok{)}
\FunctionTok{points}\NormalTok{(x\_bars, y\_bars, }\AttributeTok{col=}\StringTok{"blue"}\NormalTok{, }\AttributeTok{lwd=}\DecValTok{10}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-172-1.pdf}

The average values are much closer to the real values. That is, we can `average out' the \(u\) to get a good estimate for the effect of \(x\) on \(y\) (to get an estimate of \(\beta\)). With this understanding, we can now formalize how to compute \(\beta\) (or, to be more precise, an estimate of it: \(\hat{\beta}\)). For simplicity, we take \(\alpha=30\) as given.

In a first step we take the averages of both sides of our initial regression equation:

\(\frac{1}{N}\sum{y_i}=\frac{1}{N}\sum{(30 + \beta x_{i} + u_{i})}\),

rearranging and using the common `bar'-notation for means, we get

\(\bar{y}=30+\beta\bar{x} + \bar{u}\),

and solving for \(\beta\) and some rearranging then yields

\(\beta=\frac{\bar{y}-30-\bar{u}}{\bar{x}}\).

While the elements in \(\bar{u}\) are unobservable, we can use the rest to compute an estimate of \(\beta\):

\(\hat{\beta}=\frac{\bar{y}-30}{\bar{x}}\).

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{(}\FunctionTok{mean}\NormalTok{(y) }\SpecialCharTok{{-}}\DecValTok{30}\NormalTok{)}\SpecialCharTok{/}\FunctionTok{mean}\NormalTok{(x)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 0.9022
\end{verbatim}

\hypertarget{estimation-and-application}{%
\section{Estimation and Application}\label{estimation-and-application}}

Now that we have a basic understanding of the simple linear model, let us use this model to investigate an empirical research question based on real data. By doing so, we will take an alternative perspective to compute an estimation of the parameters of interest. The example builds on the `swiss' dataset encountered in previous chapters. First, get again familiar with the dataset:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load the data}
\FunctionTok{data}\NormalTok{(swiss)}
\CommentTok{\# look at the description}
\NormalTok{?swiss}
\end{Highlighting}
\end{Shaded}

Recall that observations in this dataset are at the province level. We use the simple linear model to better understand whether more years of schooling is improving educational outcomes. Thereby, we approximate educational attainment with the variable \texttt{Education} (the percentage of Swiss military draftees in a province that had education beyond primary school) and educational outcomes with the variable \texttt{Examination} (the percentage of draftees in a province that have received the highest mark on the standardized test as part of the army examination). We thus want to exploit that schooling systematically varies between provinces but that all military draftees need to take the same standardized examination during the military drafting process.

\hypertarget{model-specification}{%
\subsection{Model specification}\label{model-specification}}

Formally, we can express the relationship between these variables as

\(Examination_{i}= \alpha + \beta Education_{i}\),

where the parameters \(\alpha\) and \(\beta\) determine the percentage of draftees in a province that has received the highest mark, and the subscript \(i\) indicates that we model this relationship for each province \(i\) out of the \(N\) provinces (note that this also presupposes that \(\alpha\) and \(\beta\) are the same in each province). The intuitive hypothesis is that in this equation, \(\beta\) is positive, indicating that a higher share of draftees with more years of schooling results in a higher share of draftees who reach the highest examination mark.

Yet, our model arguably has a couple of weaknesses. Most importantly, from an economic point of view, it seems rather unlikely that \(Education\) is the only variable that matters for examination success. For example, it might matter a lot how students allocate their time when not in school, which might vary by province. In some provinces, children and young adults might have engaged in work activities that foster their reading and math skills outside school. In contrast, in other provinces, they might have engaged in hard manual labor in the agricultural sector. To formally acknowledge that other factors might also play a role, we extend our model with the term \(u_{i}\). For the moment, we thus subsume all other potentially relevant factors in that term:

\(Examination_{i}= \alpha + \beta Education_{i} + u_{i}\).

\hypertarget{raw-data}{%
\subsection{Raw data}\label{raw-data}}

First, look at how these two variables are jointly distributed with a scatter plot.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{plot}\NormalTok{(swiss}\SpecialCharTok{$}\NormalTok{Education, swiss}\SpecialCharTok{$}\NormalTok{Examination)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-176-1.pdf}
There seems to be a positive relationship between the years of schooling and examination success. Considering the raw data, we can formulate the problem of estimating the parameters of interest (\(\alpha\) and \(\beta\)) as the question: What is the best way to draw a straight trend line through the cloud of dots? To further specify the problem, we need to specify a criterion to judge whether such a line `fits' the data well. A rather intuitive measure is the sum of the squared differences between the line and the actual data points on the \(y\)-axis. That is, we compare what \(y\)-value we would get for a given \(x\)-value according to our trend-line (i.e., the function defining this line) with the actual \(y\)-value in the data (for simplicity, we use the general formulation with \(y\) as the dependent and \(x\) as the explanatory variable).

\hypertarget{derivation-and-implementation-of-ols-estimator}{%
\section{Derivation and implementation of OLS estimator}\label{derivation-and-implementation-of-ols-estimator}}

From the model equation, we easily see that these `differences' between the predicted and the actual values of \(y\) are the remaining unexplained component \(u\):

\(y_{i}-\hat{\alpha}-\hat{\beta} x_i=u_i\).

Hence, we want to minimize the \emph{sum of squared residuals (SSR)}: \(\sum{u_i^2}=\sum{(y_{i}-\hat{\alpha}-\hat{\beta} x_i)^2}\). Using calculus, we define the two first-order conditions:

\[\frac{\partial SSR}{\partial \hat{\alpha}}=\sum{-2(y_{i}-\hat{\alpha}-\hat{\beta} x_i)}=0\]

\[\frac{\partial SSR}{\partial \hat{\beta}}=\sum{-2x_i(y_{i}-\hat{\alpha}-\hat{\beta} x_i)}=0\]

The first condition is relatively easily solved by getting rid of the \(-2\) and considering that \(\sum{y_i}=N\bar{y}\):

\(\hat{\alpha}=\bar{y}-\hat{\beta}\bar{x}\).

By plugging the solution for \(\hat{\alpha}\) into the first order condition regarding \(\hat{\beta}\) and again considering that \(\sum{y_i}=N\bar{y}\), we get the solution for the slope coefficient estimator:

\(\frac{\sum{x_{i}y_{i}}-N\bar{y}\bar{x}}{\sum{x_i^2}-N\bar{x}^2}\).

To compute the actual estimates, we first compute \(\hat{\beta}\) and then \(\hat{\alpha}\). With all that, we can implement our OLS estimator for the simple linear regression model in R and apply it to the estimation problem at hand:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# implement the simple OLS estimator}
\CommentTok{\# verify implementation with simulated data from above}
\CommentTok{\# my\_ols(y,x)}
\CommentTok{\# should be very close to alpha=30 and beta=0.9}
\NormalTok{my\_ols }\OtherTok{\textless{}{-}}
  \ControlFlowTok{function}\NormalTok{(y,x) \{}
\NormalTok{    N }\OtherTok{\textless{}{-}} \FunctionTok{length}\NormalTok{(y)}
\NormalTok{    betahat }\OtherTok{\textless{}{-}}\NormalTok{ (}\FunctionTok{sum}\NormalTok{(y}\SpecialCharTok{*}\NormalTok{x) }\SpecialCharTok{{-}}\NormalTok{ N}\SpecialCharTok{*}\FunctionTok{mean}\NormalTok{(x)}\SpecialCharTok{*}\FunctionTok{mean}\NormalTok{(y)) }\SpecialCharTok{/}\NormalTok{ (}\FunctionTok{sum}\NormalTok{(x}\SpecialCharTok{\^{}}\DecValTok{2}\NormalTok{)}\SpecialCharTok{{-}}\NormalTok{N}\SpecialCharTok{*}\FunctionTok{mean}\NormalTok{(x)}\SpecialCharTok{\^{}}\DecValTok{2}\NormalTok{)}
\NormalTok{    alphahat }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(y)}\SpecialCharTok{{-}}\NormalTok{betahat}\SpecialCharTok{*}\FunctionTok{mean}\NormalTok{(x)}
    
    \FunctionTok{return}\NormalTok{(}\FunctionTok{list}\NormalTok{(}\AttributeTok{alpha=}\NormalTok{alphahat,}\AttributeTok{beta=}\NormalTok{betahat))}
\NormalTok{  \}}

\CommentTok{\# estimate the effect of Education on Examination}
\NormalTok{estimates }\OtherTok{\textless{}{-}} \FunctionTok{my\_ols}\NormalTok{(swiss}\SpecialCharTok{$}\NormalTok{Examination, swiss}\SpecialCharTok{$}\NormalTok{Education)}
\NormalTok{estimates}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## $alpha
## [1] 10.13
## 
## $beta
## [1] 0.5795
\end{verbatim}

Finally, we can visually inspect the estimated parameters (i.e., does the line fit well with the data?):

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{plot}\NormalTok{(swiss}\SpecialCharTok{$}\NormalTok{Education, swiss}\SpecialCharTok{$}\NormalTok{Examination)}
\FunctionTok{abline}\NormalTok{(estimates}\SpecialCharTok{$}\NormalTok{alpha, estimates}\SpecialCharTok{$}\NormalTok{beta, }\AttributeTok{col=}\StringTok{"red"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-178-1.pdf}

The fit looks rather reasonable. There seems to be indeed a positive relationship between Education and Examination. In the next step, we would likely want to know whether we can say something meaningful about how precisely this relationship is measured statistically (i.e., how `significant' \(\hat{\beta}\) is). We will leave this issue for another class.

\hypertarget{regression-toolbox-in-r}{%
\subsection{Regression toolbox in R}\label{regression-toolbox-in-r}}

When working on data analytics problems in R, there is usually no need to implement one's own estimator functions. For most data analytics problems, the basic R installation (or an additional R package) already provides easy-to-use functions with many more features than our very simple example above. For example, the work-horse function for linear regressions in R is \texttt{lm()}, which accepts regression equation expressions as \texttt{formulas} such as \texttt{Examination\textasciitilde{}Education} and a data-frame with the corresponding dataset as arguments. As a simple example, we use this function to compute the same regression estimates as before:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{estimates2 }\OtherTok{\textless{}{-}} \FunctionTok{lm}\NormalTok{(Examination}\SpecialCharTok{\textasciitilde{}}\NormalTok{Education, }\AttributeTok{data=}\NormalTok{swiss)}
\NormalTok{estimates2}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Call:
## lm(formula = Examination ~ Education, data = swiss)
## 
## Coefficients:
## (Intercept)    Education  
##      10.127        0.579
\end{verbatim}

With one additional line of code, we can compute all the common statistics about the regression estimation:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{summary}\NormalTok{(estimates2)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## 
## Call:
## lm(formula = Examination ~ Education, data = swiss)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -10.932  -4.763  -0.184   3.891  12.498 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  10.1275     1.2859    7.88  5.2e-10 ***
## Education     0.5795     0.0885    6.55  4.8e-08 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 5.77 on 45 degrees of freedom
## Multiple R-squared:  0.488,  Adjusted R-squared:  0.476 
## F-statistic: 42.9 on 1 and 45 DF,  p-value: 4.81e-08
\end{verbatim}

The t-tests displayed in this summary for the intercept (\(\alpha\)) and the slope-coefficient concerning \texttt{Education} (\(\beta\)) assess how probable it is to observe such parameter estimates if the true values of these parameters are 0 (this is one way of thinking about statistical `significance'). Specifically for the research question in this example: How often would we observe a value of \(\hat{\beta}=0.579..\) or larger if we were to draw a random sample from the same population repeatedly and if more schooling in the population does not increase the average high-success rate at the standardized examination (\(\beta=0\))? The P-values (last column) suggests that this would hardly ever be the case. If we are confident that the factors not considered in this simple model do not affect \texttt{Education}, we could conclude that \texttt{Education} has a significant and positive effect on \texttt{Examination}.

\hypertarget{data-visualization}{%
\chapter{Data Visualization}\label{data-visualization}}

\hypertarget{data-display}{%
\section{Data display}\label{data-display}}

In the last part of a data pipeline, we typically deal with data visualization and statistical results for presentation/communication. Typical output formats are reports, a thesis (BA, MA, Dissertation chapter), interactive dashboards, and websites. R (and particularly RStudio) provides a very flexible framework to manage the steps involved in visualization/presentation for all of these output formats. The first (low-level) step in preparing data/results for publication is the formatting of data values for publication. Typically, this involves some string operations to make numbers and text look nicer before we show them in a table or graph.

Consider, for example, the following summary statistics.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load packages and data}
\FunctionTok{library}\NormalTok{(tidyverse)}
\FunctionTok{data}\NormalTok{(}\StringTok{"swiss"}\NormalTok{)}
\CommentTok{\# compute summary statistics}
\NormalTok{swiss\_summary }\OtherTok{\textless{}{-}}
  \FunctionTok{summarise}\NormalTok{(swiss,}
          \AttributeTok{avg\_education =} \FunctionTok{mean}\NormalTok{(Education, }\AttributeTok{na.rm =} \ConstantTok{TRUE}\NormalTok{),}
          \AttributeTok{avg\_fertility =} \FunctionTok{mean}\NormalTok{(Fertility, }\AttributeTok{na.rm =} \ConstantTok{TRUE}\NormalTok{),}
          \AttributeTok{N =} \FunctionTok{n}\NormalTok{()}
\NormalTok{          )}
\NormalTok{swiss\_summary}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   avg_education avg_fertility  N
## 1         10.98         70.14 47
\end{verbatim}

We likely do not want to present these numbers with that many decimal places. The function \texttt{round()} can take care of this.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{swiss\_summary\_rounded }\OtherTok{\textless{}{-}} \FunctionTok{round}\NormalTok{(swiss\_summary, }\DecValTok{2}\NormalTok{)}
\NormalTok{swiss\_summary\_rounded}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   avg_education avg_fertility  N
## 1         10.98         70.14 47
\end{verbatim}

More specific formatting of numeric values is easier when coercing the numbers to character strings (text).\footnote{Note that this step only makes sense if we are sure that the numeric values won't be further analyzed or used in a plot (except for labels).} For example, depending on the audience (country/region) to which we want to communicate our results, different standards of how to format numbers are expected. In the English-speaking world it is quite common to use \texttt{.} as decimal mark, in the German-speaking world it is rather common to use \texttt{,}. The \texttt{format()}-function provides an easy way to format numbers in this way (once they are coerced to character).

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{swiss\_summary\_formatted }\OtherTok{\textless{}{-}} \FunctionTok{format}\NormalTok{(swiss\_summary\_rounded, }\AttributeTok{decimal.mark=}\StringTok{","}\NormalTok{)}
\NormalTok{swiss\_summary\_formatted}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##   avg_education avg_fertility  N
## 1         10,98         70,14 47
\end{verbatim}

R also provides various helpful functions to better format/display text strings. See, for example:

\begin{itemize}
\tightlist
\item
  Uppercase/lowercase: \texttt{toupper()}/\texttt{tolowe()}.
\item
  Remove white spaces: \texttt{trimws()},
\end{itemize}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{string }\OtherTok{\textless{}{-}} \StringTok{"AbCD "}
\FunctionTok{toupper}\NormalTok{(string)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "ABCD "
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{tolower}\NormalTok{(string)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "abcd "
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{trimws}\NormalTok{(}\FunctionTok{tolower}\NormalTok{(string))}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] "abcd"
\end{verbatim}

\hypertarget{data-visualization-with-ggplot2}{%
\section{\texorpdfstring{Data visualization with \texttt{ggplot2}}{Data visualization with ggplot2}}\label{data-visualization-with-ggplot2}}

A key technique to convincingly communicate statistical results and insights from data is visualization. How can we visualize raw data and insights gained from statistical models with R? It turns out that R is a really useful tool for data visualization, thanks to its very powerful graphics engine (i.e., the underlying low-level R functions that handle things like colors, shapes, etc.). Building on this graphics engine, there are particularly three R packages with a variety of high-level functions to plot data in R:

\begin{itemize}
\tightlist
\item
  The original \texttt{graphics} package (\citet{r_2018}; shipped with the base R installation).
\item
  The \texttt{lattice} package \citep{lattice_2008}, an implementation of the original Bell Labs `Trellis' system.
\item
  The \texttt{ggplot2} package \citep{wickham_2016}, an implementation of Leland Wilkinson's `Grammar of Graphics'.
\end{itemize}

While these packages provide well-documented high-level R functions to plot data, their syntax differs in some important ways. For R beginners, it thus makes sense to first learn how to generate plots in R with \emph{one} of these packages. Here, we focus on \texttt{ggplot2} because it is part of the \texttt{tidyverse}.

\hypertarget{grammar-of-graphics}{%
\section{`Grammar of Graphics'}\label{grammar-of-graphics}}

A few years back, statistician and computer scientist Leland Wilkinson wrote an influential book called `The Grammar of Graphics'. In this book, Wilkinson develops a formal description (`grammar') of graphics used in statistics, illustrating how different types of plots (bar plot, histogram, etc.) are special cases of an underlying framework. In short, his idea was that we can think of graphics as consisting of different design layers and thus can build and describe graphics/plots \emph{layer by layer} as illustrated below.

\begin{figure}

{\centering \includegraphics[width=0.4\linewidth]{img/gg} 

}

\caption{Illustration of the Grammar of Graphics concept.}\label{fig:gg2}
\end{figure}



This framework got implemented in R with the prominent \texttt{ggplot2}-package, building on the already powerful R graphics engine. The result is a user-friendly environment to visualize data with enormous potential to plot almost any graphic illustrating data.

\hypertarget{ggplot2-basics}{%
\subsection{\texorpdfstring{\texttt{ggplot2} basics}{ggplot2 basics}}\label{ggplot2-basics}}

Using \texttt{ggplot2} to generate a basic plot in R is quite simple. It involves three key points:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  The data must be stored in a \texttt{data.frame}/\texttt{tibble} (in tidy format).
\item
  The starting point of a plot is always the function \texttt{ggplot()}.
\item
  The first line of plot code declares the data and the `aesthetics' (e.g., which variables are mapped to the x-/y-axes):
\end{enumerate}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ my\_dataframe, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x=}\NormalTok{ xvar, }\AttributeTok{y=}\NormalTok{ yvar))}
\end{Highlighting}
\end{Shaded}

\hypertarget{tutorial}{%
\subsection{Tutorial}\label{tutorial}}

In the following, we learn the basic functionality of \texttt{ggplot} by applying it to the \texttt{swiss} dataset.

\hypertarget{loadingpreparing-the-data}{%
\subsection{Loading/preparing the data}\label{loadingpreparing-the-data}}

First, we load and inspect the data. Among other variables, it contains information about the share of inhabitants of a given Swiss province who indicate to be of Catholic faith (and not Protestant).

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# load the R package}
\FunctionTok{library}\NormalTok{(ggplot2)}
\CommentTok{\# load the data}
\FunctionTok{data}\NormalTok{(swiss)}
\CommentTok{\# get details about the data set}
\CommentTok{\# ?swiss}
\CommentTok{\# inspect the data}
\FunctionTok{head}\NormalTok{(swiss)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
##              Fertility Agriculture Examination Education
## Courtelary        80.2        17.0          15        12
## Delemont          83.1        45.1           6         9
## Franches-Mnt      92.5        39.7           5         5
## Moutier           85.8        36.5          12         7
## Neuveville        76.9        43.5          17        15
## Porrentruy        76.1        35.3           9         7
##              Catholic Infant.Mortality
## Courtelary       9.96             22.2
## Delemont        84.84             22.2
## Franches-Mnt    93.40             20.2
## Moutier         33.77             20.3
## Neuveville       5.16             20.6
## Porrentruy      90.57             26.6
\end{verbatim}

As we do not only want to use this continuous measure in the data visualization, we generate an additional factor variable called \texttt{Religion} which has either the value \texttt{\textquotesingle{}Protestant\textquotesingle{}} or \texttt{\textquotesingle{}Catholic\textquotesingle{}} depending on whether more than 50 percent of the inhabitants of the province are Catholics.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# code province as \textquotesingle{}Catholic\textquotesingle{} if more than 50\% are catholic}
\NormalTok{swiss}\SpecialCharTok{$}\NormalTok{Religion }\OtherTok{\textless{}{-}} \StringTok{\textquotesingle{}Protestant\textquotesingle{}}
\NormalTok{swiss}\SpecialCharTok{$}\NormalTok{Religion[}\DecValTok{50} \SpecialCharTok{\textless{}}\NormalTok{ swiss}\SpecialCharTok{$}\NormalTok{Catholic] }\OtherTok{\textless{}{-}} \StringTok{\textquotesingle{}Catholic\textquotesingle{}}
\NormalTok{swiss}\SpecialCharTok{$}\NormalTok{Religion }\OtherTok{\textless{}{-}} \FunctionTok{as.factor}\NormalTok{(swiss}\SpecialCharTok{$}\NormalTok{Religion)}
\end{Highlighting}
\end{Shaded}

\hypertarget{data-and-aesthetics}{%
\subsubsection{Data and aesthetics}\label{data-and-aesthetics}}

We initiate the most basic plot with \texttt{ggplot()} by defining which data to use and, in the plot aesthetics, which variable to use on the x and y axes. Here, we are interested in whether the level of education beyond primary school in a given district is related to how well draftees from the same district do in a standardized army examination (\% of draftees that get the highest mark in the examination).

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination))}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-188-1}

As we have not yet defined according to what rules the data shall be visualized, all we get is an empty `canvas' and the axes (with the respective label and ticks indicating the range of the values).

\hypertarget{geometries-type-of-plot}{%
\subsection{Geometries (\textasciitilde{} type of plot)}\label{geometries-type-of-plot}}

To actually plot the data, we have to define the `geometries', defined according to which function the data should be mapped/visualized. In other words, geometries define which `type of plot' we use to visualize the data (histogram, lines, points, etc.). In the example code below, we use \texttt{geom\_point()} to get a simple point plot.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination)) }\SpecialCharTok{+}
     \FunctionTok{geom\_point}\NormalTok{()}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-189-1}

The result indicates a positive correlation between the level of education and how well draftees do in the examination. We want to understand this correlation better. Particularly what other factors could drive this picture?

\hypertarget{facets}{%
\subsubsection{Facets}\label{facets}}

According to a popular thesis, the protestant reformation and the spread of the protestant movement in Europe were driving the development of compulsory schooling. It would thus be reasonable to hypothesize that the picture we see is partly driven by differences in schooling between Catholic and Protestant districts. To make such differences visible in the data, we use `facets' to show the same plot again, but this time separating observations from Catholic and Protestant districts:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination)) }\SpecialCharTok{+}
     \FunctionTok{geom\_point}\NormalTok{() }\SpecialCharTok{+}
     \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{Religion)}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-190-1}

Draftees from protestant districts tend to do generally better (which might indicate better primary schools or a generally stronger focus on the educational achievements of Protestant children). However, the relationship between education (beyond primary schools) and examination success seems to hold for either type of district.

\hypertarget{additional-layers-and-statistics}{%
\subsubsection{Additional layers and statistics}\label{additional-layers-and-statistics}}

Let's visualize this relationship more clearly by drawing trend lines through the scattered diagrams. Once with the non-parametric `loess'-approach and once forcing a linear model on the relationship between the two variables.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination)) }\SpecialCharTok{+}
     \FunctionTok{geom\_point}\NormalTok{() }\SpecialCharTok{+}
     \FunctionTok{geom\_smooth}\NormalTok{(}\AttributeTok{method =} \StringTok{\textquotesingle{}loess\textquotesingle{}}\NormalTok{) }\SpecialCharTok{+}
     \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{Religion)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## `geom_smooth()` using formula 'y ~ x'
\end{verbatim}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-191-1}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination)) }\SpecialCharTok{+}
     \FunctionTok{geom\_point}\NormalTok{() }\SpecialCharTok{+}
     \FunctionTok{geom\_smooth}\NormalTok{(}\AttributeTok{method =} \StringTok{\textquotesingle{}lm\textquotesingle{}}\NormalTok{) }\SpecialCharTok{+}
     \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{Religion)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## `geom_smooth()` using formula 'y ~ x'
\end{verbatim}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-192-1}

\hypertarget{additional-aesthetics}{%
\subsubsection{Additional aesthetics}\label{additional-aesthetics}}

Knowing a little about Swiss history and geography, we realize that rural cantons in mountain regions remained Catholic during the Reformation. In addition, cantonal school systems historically considered that children have to help their parents on the farms during the summers. Thus in some rural cantons, schools were closed from spring until autumn. Hence, we might want to indicate in the plot which point refers to a predominantly agricultural district. We use the aesthetics of the point geometry to color the points according to the `\texttt{Agriculture}'-variable (the \% of males involved in agriculture as an occupation).

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination)) }\SpecialCharTok{+}
     \FunctionTok{geom\_point}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\AttributeTok{color =}\NormalTok{ Agriculture)) }\SpecialCharTok{+}
     \FunctionTok{geom\_smooth}\NormalTok{(}\AttributeTok{method =} \StringTok{\textquotesingle{}lm\textquotesingle{}}\NormalTok{) }\SpecialCharTok{+}
     \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{Religion)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## `geom_smooth()` using formula 'y ~ x'
\end{verbatim}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-193-1}

The resulting picture is in line with what we have expected. Overall, the districts with a lower share of agricultural occupation tend to have higher levels of education and achievements in the examination.

\hypertarget{coordinatesthemes-fine-tuning-the-plot}{%
\subsubsection{Coordinates/Themes: Fine-tuning the plot}\label{coordinatesthemes-fine-tuning-the-plot}}

Finally, there are countless options to refine the plot further. For example, we can easily change the orientation/coordinates of the plot:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination)) }\SpecialCharTok{+}
     \FunctionTok{geom\_point}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\AttributeTok{color =}\NormalTok{ Agriculture)) }\SpecialCharTok{+}
     \FunctionTok{geom\_smooth}\NormalTok{(}\AttributeTok{method =} \StringTok{\textquotesingle{}lm\textquotesingle{}}\NormalTok{) }\SpecialCharTok{+}
     \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{Religion) }\SpecialCharTok{+}
     \FunctionTok{coord\_flip}\NormalTok{()}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## `geom_smooth()` using formula 'y ~ x'
\end{verbatim}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-194-1}

In addition, the \texttt{theme()}-function allows changing almost every aspect of the plot (margins, font face, font size, etc.). For example, we might prefer to have the plot legend at the bottom and have larger axis labels.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination)) }\SpecialCharTok{+}
     \FunctionTok{geom\_point}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\AttributeTok{color =}\NormalTok{ Agriculture)) }\SpecialCharTok{+}
     \FunctionTok{geom\_smooth}\NormalTok{(}\AttributeTok{method =} \StringTok{\textquotesingle{}lm\textquotesingle{}}\NormalTok{) }\SpecialCharTok{+}
     \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{Religion) }\SpecialCharTok{+}
     \FunctionTok{theme}\NormalTok{(}\AttributeTok{legend.position =} \StringTok{"bottom"}\NormalTok{, }\AttributeTok{axis.text=}\FunctionTok{element\_text}\NormalTok{(}\AttributeTok{size=}\DecValTok{12}\NormalTok{) )}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## `geom_smooth()` using formula 'y ~ x'
\end{verbatim}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-195-1}

Moreover, several theme templates offer ready-made designs for plots:

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination)) }\SpecialCharTok{+}
     \FunctionTok{geom\_point}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\AttributeTok{color =}\NormalTok{ Agriculture)) }\SpecialCharTok{+}
     \FunctionTok{geom\_smooth}\NormalTok{(}\AttributeTok{method =} \StringTok{\textquotesingle{}lm\textquotesingle{}}\NormalTok{) }\SpecialCharTok{+}
     \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{Religion) }\SpecialCharTok{+}
     \FunctionTok{theme\_minimal}\NormalTok{()}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## `geom_smooth()` using formula 'y ~ x'
\end{verbatim}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-196-1}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(}\AttributeTok{data =}\NormalTok{ swiss, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{x =}\NormalTok{ Education, }\AttributeTok{y =}\NormalTok{ Examination)) }\SpecialCharTok{+}
     \FunctionTok{geom\_point}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\AttributeTok{color =}\NormalTok{ Agriculture)) }\SpecialCharTok{+}
     \FunctionTok{geom\_smooth}\NormalTok{(}\AttributeTok{method =} \StringTok{\textquotesingle{}lm\textquotesingle{}}\NormalTok{) }\SpecialCharTok{+}
     \FunctionTok{facet\_wrap}\NormalTok{(}\SpecialCharTok{\textasciitilde{}}\NormalTok{Religion) }\SpecialCharTok{+}
     \FunctionTok{theme\_dark}\NormalTok{()}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## `geom_smooth()` using formula 'y ~ x'
\end{verbatim}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-197-1}

\hypertarget{dynamic-documents}{%
\section{Dynamic documents}\label{dynamic-documents}}

Dynamic documents are a way to directly/dynamically integrate the results of an analysis in R (numbers, tables, plots) in written text (a report, thesis, slide set, website, etc.). That is, we can write a report in the so-called `R-Markdown' format and place it directly in the same document `chunks' of R code, which we want to be executed each time we `knit' the report. Knitting the document means that the following steps are executed under the hood:

\begin{enumerate}
\def\labelenumi{\arabic{enumi}.}
\tightlist
\item
  The code in the R chunks is executed, and the results are cached and formatted for print.
\item
  The formatted R output is embedded in a so-called `Markdown'-file (\texttt{.md}).
\item
  The markdown file is rendered as either a PDF, HTML, or Word file (with additional formatting options, depending on the output format).
\end{enumerate}

The entire procedure of importing, cleaning, analyzing, and visualizing data can thus be combined in one document, based on which we can generate a meaningful output to communicate our results.

\hypertarget{tutorial-how-to-give-the-wrong-impression-with-data-visualization}{%
\section{Tutorial: How to give the wrong impression with data visualization}\label{tutorial-how-to-give-the-wrong-impression-with-data-visualization}}

\hypertarget{background-and-aim}{%
\subsection{Background and aim}\label{background-and-aim}}

Referring to the supposed trade-off between public health and economic performance in the context of the SARS-CoV-2 pandemic, Georgia Governor Brian Kemp has ``reopened the state on April 24 {[}2020{]}, citing a downward trend of COVID-19 cases being reported by the Georgia Department of Health {[}\ldots{]} Georgians have been relying on this data (number of cases, hospitalizations and deaths in their areas) to determine whether or not it's safe for them to go out.'' (\href{https://www.firstcoastnews.com/article/news/local/georgia/georgia-data-numbers-misrepresented/77-08c31538-3f26-4348-9f30-d81b11dd4d24}{FIRSTCOAST NEWS}) It later turned out that there was no downward trend at all but that the data on Covid cases was intentionally visualized in such a way to give the impression of a downward trend. In this tutorial, we aim to replicate the doctored downward trend visualization, using the original data and \texttt{ggplot2}.\footnote{Note that the original data is not 100\% identical with the data used in the original plot. The reason is that Covid-numbers are updated/corrected over time, and the original plot uses very recent data.}

\includegraphics[width=300pt]{img/georgia_vis}

\hypertarget{data-import-and-preparation}{%
\subsection{Data import and preparation}\label{data-import-and-preparation}}

The original raw data is provided in a zip-file (compressed file). The code chunk below shows some tricks of how to download and import data from an online zip-file as part of a simple data analytics script. First, we generate a temporary file with \texttt{tempfile()}. We then download the compressed zip file containing the Covid-data to the temporary file via \texttt{download.file(DATA\_URL,\ destfile\ =\ tmp)}. Finally, we use \texttt{unzip(tmp,\ files\ =\ "epicurve\_rpt\_date.csv"\ )} in order to decompress the one CSV-file in the zip-file we want to work with. After that, we can simply import the CSV-file the usual way (with the \texttt{read\_cav()} function).

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# SET UP {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\CommentTok{\# load packages}
\FunctionTok{library}\NormalTok{(tidyverse)}
\FunctionTok{library}\NormalTok{(ggplot2)}

\CommentTok{\# fix vars}
\NormalTok{DATA\_URL }\OtherTok{\textless{}{-}} \StringTok{"https://ga{-}covid19.ondemand.sas.com/docs/ga\_covid\_data.zip"}
\NormalTok{COUNTIES }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\StringTok{"Cobb"}\NormalTok{, }\StringTok{"DeKalb"}\NormalTok{, }\StringTok{"Fulton"}\NormalTok{, }\StringTok{"Gwinnett"}\NormalTok{, }\StringTok{"Hall"}\NormalTok{)}
\NormalTok{MIN\_DATE }\OtherTok{\textless{}{-}} \FunctionTok{as.Date}\NormalTok{(}\StringTok{"2020{-}04{-}26"}\NormalTok{)}
\NormalTok{MAX\_DATE }\OtherTok{\textless{}{-}} \FunctionTok{as.Date}\NormalTok{(}\StringTok{"2020{-}05{-}09"}\NormalTok{)}


\CommentTok{\# FETCH/IMPORT DATA {-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}{-}}

\CommentTok{\# create a temporary file to store the downloaded zipfile}
\NormalTok{tmp }\OtherTok{\textless{}{-}} \FunctionTok{tempfile}\NormalTok{()}
\FunctionTok{download.file}\NormalTok{(DATA\_URL, }\AttributeTok{destfile =}\NormalTok{ tmp)}
\CommentTok{\# unzip the file we need}
\NormalTok{path }\OtherTok{\textless{}{-}} \FunctionTok{unzip}\NormalTok{(tmp, }\AttributeTok{files =} \StringTok{"epicurve\_rpt\_date.csv"}\NormalTok{ )}
\CommentTok{\# read the data}
\NormalTok{cases }\OtherTok{\textless{}{-}} \FunctionTok{read\_csv}\NormalTok{(path)}
\CommentTok{\# inspect the data}
\FunctionTok{head}\NormalTok{(cases)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## # A tibble: 6 x 20
##   measure     county report_date cases deaths cases_cum death_cum
##   <chr>       <chr>  <date>      <dbl>  <dbl>     <dbl>     <dbl>
## 1 state_total Georg~ 2020-02-01      0      0         0         0
## 2 state_total Georg~ 2020-02-02      0      0         0         0
## 3 state_total Georg~ 2020-02-03      0      0         0         0
## 4 state_total Georg~ 2020-02-04      0      0         0         0
## 5 state_total Georg~ 2020-02-05      0      0         0         0
## 6 state_total Georg~ 2020-02-06      0      0         0         0
## # ... with 13 more variables: moving_avg_cases <dbl>,
## #   moving_avg_deaths <dbl>, antigen_cases <dbl>,
## #   probable_deaths <dbl>, antigen_case_hospitalization <dbl>,
## #   confirmed_case_hospitalization <dbl>,
## #   antigen_cases_cum <dbl>, probable_deaths_cum <dbl>,
## #   total_cases <dbl>, total_cases_cum <dbl>,
## #   moving_avg_antigen_cases <dbl>, ...
\end{verbatim}

Once the data is imported, we select/filter the part of the dataset used in the original bar plot. Note the tidyverse-functions introduced in previous lectures to prepare an analytic dataset out of the raw data efficiently. At one point, we use the \texttt{order()}-function to order the observations according to the report date of covid cases. This is done when displaying the frequency of cases over time. This way, in the following plot, the x-axis serves as a time axis, displaying the date of the corresponding reported case numbers from the beginning of the observation period on the left to the end of the observation period on the right).

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# only observations in the five major counties and during the relevant days}
\NormalTok{cases }\OtherTok{\textless{}{-}} \FunctionTok{filter}\NormalTok{(cases, county }\SpecialCharTok{\%in\%}\NormalTok{ COUNTIES, MIN\_DATE }\SpecialCharTok{\textless{}=}\NormalTok{ report\_date, report\_date }\SpecialCharTok{\textless{}=}\NormalTok{ MAX\_DATE)}
\CommentTok{\# only the relevant variables}
\NormalTok{cases }\OtherTok{\textless{}{-}} \FunctionTok{select}\NormalTok{(cases, county, report\_date, cases)}
\CommentTok{\# order according to date}
\NormalTok{cases }\OtherTok{\textless{}{-}}\NormalTok{ cases[}\FunctionTok{order}\NormalTok{(cases}\SpecialCharTok{$}\NormalTok{report\_date),]}
\CommentTok{\# we add an additional column in which we treat dates as categories}
\NormalTok{cases }\OtherTok{\textless{}{-}} \FunctionTok{mutate}\NormalTok{(cases, }\AttributeTok{date=}\FunctionTok{factor}\NormalTok{(report\_date))}
\end{Highlighting}
\end{Shaded}

\hypertarget{plot}{%
\subsection{Plot}\label{plot}}

First, we show what the actual (honest) plot would essentially look like.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(cases, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{y=}\NormalTok{cases, }\AttributeTok{x=}\NormalTok{date, }\AttributeTok{fill=}\NormalTok{county)) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{(}\AttributeTok{position =} \FunctionTok{position\_dodge2}\NormalTok{(), }\AttributeTok{stat =} \StringTok{"identity"}\NormalTok{) }\SpecialCharTok{+}
  \FunctionTok{scale\_x\_discrete}\NormalTok{(}\AttributeTok{guide =} \FunctionTok{guide\_axis}\NormalTok{(}\AttributeTok{n.dodge =} \DecValTok{2}\NormalTok{)) }\CommentTok{\# this avoids overlapping of x{-}axis labels (and is similar to what is done in the original plot)}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-201-1}

There is no obvious downward trend visible. Now, let us look at what steps are involved in manipulating the visualization in order to give the wrong impression of a downward trend. Importantly, all the manipulation is purely done via the data plotting. We do not touch/manipulate the underlying data (such as removing observations or falsifying numbers).

Two things become apparent when comparing the original plot with what we have so far. First, the order of days on the X-axis is not chronological but seems to be based on the total number of cases per day. Second, the order of the bars of a given day does not follow the alphabetical order of the county names but the number of cases in each of the counties. Let's address the two aspects one at the time. To change the order on the x-axis, we have to re-order the factor levels in \texttt{date}.

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{cases2 }\OtherTok{\textless{}{-}} \FunctionTok{mutate}\NormalTok{(cases, }\AttributeTok{date=} \FunctionTok{fct\_reorder}\NormalTok{(date, cases, sum,}\AttributeTok{.desc =} \ConstantTok{TRUE}\NormalTok{)) }\CommentTok{\# re{-}order the dates based on the total number of cases per date.}

\FunctionTok{ggplot}\NormalTok{(cases2, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{y=}\NormalTok{cases, }\AttributeTok{x=}\NormalTok{date, }\AttributeTok{fill=}\NormalTok{county)) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{(}\AttributeTok{position =} \FunctionTok{position\_dodge2}\NormalTok{(), }\AttributeTok{stat =} \StringTok{"identity"}\NormalTok{) }\SpecialCharTok{+}
  \FunctionTok{scale\_x\_discrete}\NormalTok{(}\AttributeTok{guide =} \FunctionTok{guide\_axis}\NormalTok{(}\AttributeTok{n.dodge =} \DecValTok{2}\NormalTok{))  }\CommentTok{\# this avoids overlapping of x{-}axis labels (and is similar to what is done in the original plot)}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-202-1}

Note that the number of cases is not exactly the same as in the original. Quite the reason for this is that the numbers have been updated during May. Given that the overall pattern is very similar, there is no reason to believe that the actual numbers underlying the original figure had been manipulated too. Now, let us address the second aspect (ordering of bars per date). For this, we use the \texttt{group} aesthetic, indicating to \texttt{ggplot} that we want the number of cases to be used to order the bars within each point in time (date). Setting \texttt{position\_dodge2(reverse=TRUE)} simply means we want the bars per date to be ordered in decreasing order (per default, it would be increasing; try out the difference by changing it to \texttt{position\_dodge2(reverse=FALSE)} or simply \texttt{position\_dodge2()}).

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(cases2, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{y=}\NormalTok{cases, }\AttributeTok{x=}\NormalTok{date, }\AttributeTok{fill=}\NormalTok{county, }\AttributeTok{group=}\NormalTok{cases)) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\AttributeTok{group=}\NormalTok{cases), }\AttributeTok{position =} \FunctionTok{position\_dodge2}\NormalTok{(}\AttributeTok{reverse=}\ConstantTok{TRUE}\NormalTok{), }\AttributeTok{stat =} \StringTok{"identity"}\NormalTok{) }\SpecialCharTok{+}
    \FunctionTok{scale\_x\_discrete}\NormalTok{(}\AttributeTok{guide =} \FunctionTok{guide\_axis}\NormalTok{(}\AttributeTok{n.dodge =} \DecValTok{2}\NormalTok{))  }\CommentTok{\# this avoids overlapping of x{-}axis labels (and is similar to what is done in the original plot)}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-203-1}

This already looks much more like the original plot.

\hypertarget{cosmetics-theme}{%
\subsection{Cosmetics: theme}\label{cosmetics-theme}}

Finally, we tweaked the plot's theme to make it look more similar to the original. The following code is a first shot at addressing the most obvious aspects to make the plot more similar to the original. More steps might be needed to make it essentially identical (consider, for example, the color scheme of the bars). This part of the tutorial is a nice illustration of how versatile the \texttt{ggplot2}-\texttt{theme()}-function is to tweak every cosmetic detail of a plot.

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{ggplot}\NormalTok{(cases2, }\FunctionTok{aes}\NormalTok{(}\AttributeTok{y=}\NormalTok{cases, }\AttributeTok{x=}\NormalTok{date, }\AttributeTok{fill=}\NormalTok{county, }\AttributeTok{group=}\NormalTok{cases)) }\SpecialCharTok{+}
  \FunctionTok{geom\_bar}\NormalTok{(}\FunctionTok{aes}\NormalTok{(}\AttributeTok{group=}\NormalTok{cases), }\AttributeTok{position =} \FunctionTok{position\_dodge2}\NormalTok{(}\AttributeTok{reverse=}\ConstantTok{TRUE}\NormalTok{), }\AttributeTok{stat =} \StringTok{"identity"}\NormalTok{) }\SpecialCharTok{+}
  \FunctionTok{ggtitle}\NormalTok{(}\StringTok{"Top5 Counties with the Greatest Number of Confirmed COVID{-}19 Cases"}\NormalTok{) }\SpecialCharTok{+}
  \FunctionTok{scale\_x\_discrete}\NormalTok{(}\AttributeTok{guide =} \FunctionTok{guide\_axis}\NormalTok{(}\AttributeTok{n.dodge =} \DecValTok{2}\NormalTok{)) }\SpecialCharTok{+}  \CommentTok{\# this avoids overlapping of x{-}axis labels (and is similar to what is done in the original plot)}
  \FunctionTok{guides}\NormalTok{(}\AttributeTok{fill=}\FunctionTok{guide\_legend}\NormalTok{(}\AttributeTok{title =} \StringTok{"County"}\NormalTok{,}
                           \AttributeTok{title.position =} \StringTok{"top"}\NormalTok{,}
                           \AttributeTok{direction =} \StringTok{"horizontal"}\NormalTok{,}
                           \AttributeTok{title.hjust =} \FloatTok{0.5}\NormalTok{)) }\SpecialCharTok{+}
  \FunctionTok{theme}\NormalTok{(}\AttributeTok{legend.position =} \StringTok{"top"}\NormalTok{,}
        \AttributeTok{plot.title =} \FunctionTok{element\_text}\NormalTok{(}\AttributeTok{colour =} \StringTok{"white"}\NormalTok{, }\AttributeTok{hjust =} \SpecialCharTok{{-}}\FloatTok{0.1}\NormalTok{),}
        \AttributeTok{legend.key.size =} \FunctionTok{unit}\NormalTok{(}\FloatTok{0.5}\NormalTok{,}\StringTok{"line"}\NormalTok{),}
        \AttributeTok{panel.grid.major =} \FunctionTok{element\_blank}\NormalTok{(),}
        \AttributeTok{panel.grid.minor =} \FunctionTok{element\_blank}\NormalTok{(),        }
        \AttributeTok{panel.background =} \FunctionTok{element\_rect}\NormalTok{(}\AttributeTok{fill=}\StringTok{"\#132a52"}\NormalTok{),}
        \AttributeTok{legend.background =} \FunctionTok{element\_rect}\NormalTok{(}\AttributeTok{fill=}\StringTok{"\#132a52"}\NormalTok{),}
        \AttributeTok{plot.background =} \FunctionTok{element\_rect}\NormalTok{(}\AttributeTok{fill=}\StringTok{"\#132a52"}\NormalTok{),}
        \AttributeTok{legend.text =} \FunctionTok{element\_text}\NormalTok{(}\AttributeTok{colour =} \StringTok{"\#536b8d"}\NormalTok{),}
        \AttributeTok{legend.title =} \FunctionTok{element\_text}\NormalTok{(}\AttributeTok{colour =} \StringTok{"white"}\NormalTok{),}
        \AttributeTok{axis.text =} \FunctionTok{element\_text}\NormalTok{(}\AttributeTok{colour =} \StringTok{"\#536b8d"}\NormalTok{),}
        \AttributeTok{axis.ticks =} \FunctionTok{element\_line}\NormalTok{(}\AttributeTok{colour =} \StringTok{"\#71869e"}\NormalTok{),}
        \AttributeTok{axis.line =} \FunctionTok{element\_line}\NormalTok{(}\AttributeTok{colour=}\StringTok{"\#71869e"}\NormalTok{),}
        \AttributeTok{axis.title =} \FunctionTok{element\_blank}\NormalTok{())}
\end{Highlighting}
\end{Shaded}

\includegraphics[width=0.85\linewidth]{datahandling_files/figure-latex/unnamed-chunk-204-1}

\backmatter

\hypertarget{appendix-appendix}{%
\appendix \addcontentsline{toc}{chapter}{\appendixname}}


\hypertarget{appendix-a}{%
\chapter{Appendix A}\label{appendix-a}}

\hypertarget{understanding-statistics-and-probability-with-code}{%
\section{Understanding statistics and probability with code}\label{understanding-statistics-and-probability-with-code}}

\hypertarget{random-draws-and-distributions}{%
\subsection{Random draws and distributions}\label{random-draws-and-distributions}}

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{normal\_distr }\OtherTok{\textless{}{-}} \FunctionTok{rnorm}\NormalTok{(}\DecValTok{1000}\NormalTok{)}
\FunctionTok{hist}\NormalTok{(normal\_distr)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-205-1.pdf}

\hypertarget{illustration-of-variability}{%
\subsection{Illustration of variability}\label{illustration-of-variability}}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# draw a random sample from a normal distribution with a large standard deviation}
\NormalTok{largevar }\OtherTok{\textless{}{-}} \FunctionTok{rnorm}\NormalTok{(}\DecValTok{10000}\NormalTok{, }\AttributeTok{mean =} \DecValTok{5000}\NormalTok{, }\AttributeTok{sd =} \DecValTok{5}\NormalTok{)}
\CommentTok{\# draw a random sample from a normal distribution with a small standard deviation}
\NormalTok{littlevar }\OtherTok{\textless{}{-}} \FunctionTok{rnorm}\NormalTok{(}\DecValTok{10000}\NormalTok{, }\AttributeTok{mean =} \DecValTok{5000}\NormalTok{, }\AttributeTok{sd =} \DecValTok{1}\NormalTok{)}

\CommentTok{\# visualize the distributions of both samples with a density plot}
\FunctionTok{plot}\NormalTok{(}\FunctionTok{density}\NormalTok{(littlevar), }\AttributeTok{col =} \StringTok{"blue"}\NormalTok{, }
     \AttributeTok{xlim=}\FunctionTok{c}\NormalTok{(}\FunctionTok{min}\NormalTok{(largevar), }\FunctionTok{max}\NormalTok{(largevar)), }\AttributeTok{main=}\StringTok{"Income Distribution"}\NormalTok{)}
\FunctionTok{lines}\NormalTok{(}\FunctionTok{density}\NormalTok{(largevar), }\AttributeTok{col =} \StringTok{"red"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-206-1.pdf}

\textbf{Note:} the red curve illustrates the distribution of the sample with a large standard deviation (a lot of variability) whereas the blue curve illustrates the one with a rather small standard deviation.

\hypertarget{skewness-and-kurtosis}{%
\subsection{Skewness and Kurtosis}\label{skewness-and-kurtosis}}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Install the R{-}package called "moments" with the following command (if not installed yet):}
\CommentTok{\# install.packages("moments")}

\CommentTok{\# load the package}
\FunctionTok{library}\NormalTok{(moments)}
\end{Highlighting}
\end{Shaded}

\hypertarget{skewness}{%
\subsubsection{Skewness}\label{skewness}}

Skewness refers to how symmetric the frequency distribution of a variable is. For example, a distribution can be `positively skewed' meaning it has a long tail on the right and a lot of `mass' (observations) on the left. We can see that when visualizing the distribution in a histogram or a density plot. In R this looks as follow (consider the comments in the code explaining what each line does):

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# draw a random sample of simulated data from a normal distribution}
\CommentTok{\# the sample is of size 1000 (hence, n = 1000)}
\NormalTok{sample }\OtherTok{\textless{}{-}} \FunctionTok{rnorm}\NormalTok{(}\AttributeTok{n =} \DecValTok{1000}\NormalTok{)}

\CommentTok{\# plot a histogram and a density plot of that sample}
\CommentTok{\# note that the distribution is neither strongly positively nor negatively skewed}
\CommentTok{\# (this is to be expected, as we have drawn a sample from a normal distribution)}
\FunctionTok{hist}\NormalTok{(sample)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-208-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{plot}\NormalTok{(}\FunctionTok{density}\NormalTok{(sample))}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-208-2.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# now compute the skewness}
\FunctionTok{skewness}\NormalTok{(sample)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] -0.01701
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Now we intentionally change our sample to be strongly positively skewed}
\CommentTok{\# We do that by adding some outliers (observations with very high values) to the sample }
\NormalTok{sample }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(sample, (}\FunctionTok{rnorm}\NormalTok{(}\DecValTok{200}\NormalTok{) }\SpecialCharTok{+} \DecValTok{2}\NormalTok{), (}\FunctionTok{rnorm}\NormalTok{(}\DecValTok{200}\NormalTok{) }\SpecialCharTok{+} \DecValTok{3}\NormalTok{))}

\CommentTok{\# Have a look at the distribution and re{-}calculate the skewness}
\FunctionTok{plot}\NormalTok{(}\FunctionTok{density}\NormalTok{(sample))}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-208-3.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{skewness}\NormalTok{(sample)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 0.5001
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\#}
\end{Highlighting}
\end{Shaded}

\hypertarget{kurtosis}{%
\subsubsection{Kurtosis}\label{kurtosis}}

Kurtosis refers to how much `mass' a distribution has in its `tails'. It thus tells us something about whether a distribution tends to have a lot of outliers. Again, plotting the data can help us understand this concept of kurtosis. Lets have a look at this in R (consider the comments in the code explaining what each line does):

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# draw a random sample of simulated data from a normal distribution}
\CommentTok{\# the sample is of size 1000 (hence, n = 1000)}
\NormalTok{sample }\OtherTok{\textless{}{-}} \FunctionTok{rnorm}\NormalTok{(}\AttributeTok{n =} \DecValTok{1000}\NormalTok{)}

\CommentTok{\# plot the density \& compute the kurtosis}
\FunctionTok{plot}\NormalTok{(}\FunctionTok{density}\NormalTok{(sample))}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-209-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{kurtosis}\NormalTok{(sample)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 2.96
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# now lets remove observations from the extremes in this distribution}
\CommentTok{\# we thus intentionally alter the distribution to have less mass in its tails}
\NormalTok{sample }\OtherTok{\textless{}{-}}\NormalTok{ sample[ sample }\SpecialCharTok{\textgreater{}} \SpecialCharTok{{-}}\FloatTok{0.6} \SpecialCharTok{\&}\NormalTok{ sample }\SpecialCharTok{\textless{}} \FloatTok{0.6}\NormalTok{]}

\CommentTok{\# plot the distribution again and see how the tails of it (and thus the kurtosis) has changed}
\FunctionTok{plot}\NormalTok{(}\FunctionTok{density}\NormalTok{(sample))}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/unnamed-chunk-209-2.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# re{-}calculate the kurtosis}
\FunctionTok{kurtosis}\NormalTok{(sample)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 1.846
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# as expected, the kurtosis has now a lower value}
\end{Highlighting}
\end{Shaded}

\hypertarget{implement-the-formulas-for-skewness-and-kurtosis-in-r}{%
\subsubsection{Implement the formulas for skewness and kurtosis in R}\label{implement-the-formulas-for-skewness-and-kurtosis-in-r}}

\textbf{Skewness}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# own implementation}
\FunctionTok{sum}\NormalTok{((sample}\SpecialCharTok{{-}}\FunctionTok{mean}\NormalTok{(sample))}\SpecialCharTok{\^{}}\DecValTok{3}\NormalTok{) }\SpecialCharTok{/}\NormalTok{ ((}\FunctionTok{length}\NormalTok{(sample)}\SpecialCharTok{{-}}\DecValTok{1}\NormalTok{) }\SpecialCharTok{*} \FunctionTok{sd}\NormalTok{(sample)}\SpecialCharTok{\^{}}\DecValTok{3}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 0.01479
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# implementation in moments package}
\FunctionTok{skewness}\NormalTok{(sample)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 0.01481
\end{verbatim}

\textbf{Kurtosis}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# own implementation}
\FunctionTok{sum}\NormalTok{((sample}\SpecialCharTok{{-}}\FunctionTok{mean}\NormalTok{(sample))}\SpecialCharTok{\^{}}\DecValTok{4}\NormalTok{) }\SpecialCharTok{/}\NormalTok{ ((}\FunctionTok{length}\NormalTok{(sample)}\SpecialCharTok{{-}}\DecValTok{1}\NormalTok{) }\SpecialCharTok{*} \FunctionTok{sd}\NormalTok{(sample)}\SpecialCharTok{\^{}}\DecValTok{4}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 1.842
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# implementation in moments package}
\FunctionTok{kurtosis}\NormalTok{(sample)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 1.846
\end{verbatim}

\hypertarget{the-law-of-large-numbers}{%
\subsection{The Law of Large Numbers}\label{the-law-of-large-numbers}}

The Law of Large Numbers (LLN) is an important statistical property which essentially describes how the behavior of sample averages is related to sample size. Particularly, it states that the sample mean can come as close as we like to the mean of the population from which it is drawn by simply increasing the sample size. That is, the larger our randomly selected sample from a population, the closer is that sample's mean to the mean of the population.

Think of playing dice. Each time we roll a fair die, the result is either 1, 2, 3, 4, 5, or 6, whereby each possible outcome can occur with the same probability (1/6). In other words we randomly draw die-values. Thus we can expect that the average of the resulting die values is (1 + 2 + 3 + 4 + 5 + 6)/6 = 3.5.

We can investigate this empirically: We roll a fair die once and record the result. We roll it again, and again we record the result. We keep rolling the die and recording results until we get 100 recorded results. Intuitively, we would expect to observe each possible die-value about equally often (given that the die is fair) because each time we roll the die, each possible value (1,2,..,6) is equally likely to be the result. And we would thus expect the average of the resulting die values to be around 3.5. However, just by chance it can obviously be the case that one value (say 5) occurs slightly more often than another value (say 1), leading to a sample mean slightly larger than 3.5. In this context, the LLN states that by increasing the number of times we are rolling the die, we will get closer and closer to 3.5. Now, let's implement this experiment in R:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# first we define the potential values a die can take}
\NormalTok{dvalues }\OtherTok{\textless{}{-}} \DecValTok{1}\SpecialCharTok{:}\DecValTok{6} \CommentTok{\# the : operator generates a regular sequence of numbers (from:to)}
\NormalTok{dvalues}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 1 2 3 4 5 6
\end{verbatim}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# define the size of the sample n (how often do we roll the die...)}
\CommentTok{\# for a start, we only roll the die ten times}
\NormalTok{n }\OtherTok{\textless{}{-}} \DecValTok{10}
\CommentTok{\# draw the random sample: \textquotesingle{}roll the die n times and record each result\textquotesingle{}}
\NormalTok{results }\OtherTok{\textless{}{-}} \FunctionTok{sample}\NormalTok{( }\AttributeTok{x =}\NormalTok{ dvalues, }\AttributeTok{size =}\NormalTok{ n, }\AttributeTok{replace =} \ConstantTok{TRUE}\NormalTok{)}
\CommentTok{\# compute the mean}
\FunctionTok{mean}\NormalTok{(results)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 3.6
\end{verbatim}

As you can see we are relatively close to 3.5, but not quite there. So let's roll the die more often and calculate the mean of the resulting values again:

\begin{Shaded}
\begin{Highlighting}[]
\NormalTok{n }\OtherTok{\textless{}{-}} \DecValTok{100}
\CommentTok{\# draw the random sample: \textquotesingle{}roll the die n times and record each result\textquotesingle{}}
\NormalTok{results }\OtherTok{\textless{}{-}} \FunctionTok{sample}\NormalTok{( }\AttributeTok{x =}\NormalTok{ dvalues, }\AttributeTok{size =}\NormalTok{ n, }\AttributeTok{replace =} \ConstantTok{TRUE}\NormalTok{)}
\CommentTok{\# compute the mean}
\FunctionTok{mean}\NormalTok{(results)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 3.69
\end{verbatim}

We are already close to 3.5! Now let's scale up these comparisons and show how the sample means are getting even closer to 3.5 when increasing the number of times we roll the die up to 10'000.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# Essentially, what we are doing here is repeating the experiment above many times, }
\CommentTok{\# each time increasing n.}
\CommentTok{\# define the set of sample sizes}
\NormalTok{ns }\OtherTok{\textless{}{-}} \FunctionTok{seq}\NormalTok{(}\AttributeTok{from =} \DecValTok{10}\NormalTok{, }\AttributeTok{to =} \DecValTok{10000}\NormalTok{, }\AttributeTok{by =} \DecValTok{10}\NormalTok{)}
\CommentTok{\# initiate an empty list to record the results}
\NormalTok{means }\OtherTok{\textless{}{-}} \FunctionTok{list}\NormalTok{()}
\FunctionTok{length}\NormalTok{(means) }\OtherTok{\textless{}{-}} \FunctionTok{length}\NormalTok{(ns)}
\CommentTok{\# iterate through each sample size: \textquotesingle{}repeat the die experiment for each sample size\textquotesingle{}}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in} \DecValTok{1}\SpecialCharTok{:}\FunctionTok{length}\NormalTok{(ns)) \{}
     
\NormalTok{     means[[i]] }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(}\FunctionTok{sample}\NormalTok{( }\AttributeTok{x =}\NormalTok{ dvalues,}
                                \AttributeTok{size =}\NormalTok{ ns[i],}
                                \AttributeTok{replace =} \ConstantTok{TRUE}\NormalTok{))}
\NormalTok{\}}

\CommentTok{\# visualize the result: plot sample means against sample size}
\FunctionTok{plot}\NormalTok{(ns, }\FunctionTok{unlist}\NormalTok{(means),}
     \AttributeTok{ylab =} \StringTok{"Sample Mean"}\NormalTok{,}
     \AttributeTok{xlab =} \StringTok{"Sample Size"}\NormalTok{,}
     \AttributeTok{pch =} \DecValTok{16}\NormalTok{,}
     \AttributeTok{cex =}\NormalTok{ .}\DecValTok{6}\NormalTok{)}
\FunctionTok{abline}\NormalTok{(}\AttributeTok{h =} \FloatTok{3.5}\NormalTok{, }\AttributeTok{col =} \StringTok{"red"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/lln-1.pdf}

We observe that with smaller sample sizes the sample means are broadly spread around the population mean of 3.5. However, the more we go to the right extreme of the x-axis (and thus the larger the sample size), the narrower the sample means are spread around the population mean.

\hypertarget{the-central-limit-theorem}{%
\subsection{The Central Limit Theorem}\label{the-central-limit-theorem}}

The Central Limit Theorem (CLT) is an almost miraculous statistical property enabling us to test the statistical significance of a statistic such as the mean. In essence, the CLT states that as long as we have a large enough sample, the t-statistic (applied, e.g., to test whether the mean is equal to a particular value) is approximately standard normal distributed. This holds independently of how the underlying data is distributed.

Consider the dice-play-example above. We might want to statistically test whether we are indeed playing with a fair die. In order to test that we would roll the die 100 times and record each resulting value. We would then compute the sample mean and standard deviation in order to assess how likely it was to observe the mean we observe if the population mean actually is 3.5 (thus our H0 would be pop\_mean = 3.5, or in plain English `the die is fair'). However, the distribution of the resulting die values are not standard normal distributed. So how can we interpret the sample standard deviation and the sample mean in the context of our hypothesis?

The simplest way to interpret these measures is by means of a \emph{t-statistic}. A t-statistic for the sample mean under our working hypothesis that pop\_mean = 3.5 is constructed as \texttt{t(3.5)\ =\ (sample\_mean\ -\ 3.5)\ /\ (sample\_sd/sqrt(n))}. Let's illustrate this in R:

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# First we roll the die like above}
\NormalTok{n }\OtherTok{\textless{}{-}} \DecValTok{100}
\CommentTok{\# draw the random sample: \textquotesingle{}roll the die n times and record each result\textquotesingle{}}
\NormalTok{results }\OtherTok{\textless{}{-}} \FunctionTok{sample}\NormalTok{( }\AttributeTok{x =}\NormalTok{ dvalues, }\AttributeTok{size =}\NormalTok{ n, }\AttributeTok{replace =} \ConstantTok{TRUE}\NormalTok{)}
\CommentTok{\# compute the mean}
\NormalTok{sample\_mean }\OtherTok{\textless{}{-}} \FunctionTok{mean}\NormalTok{(results)}
\CommentTok{\# compute the sample SD}
\NormalTok{sample\_sd }\OtherTok{\textless{}{-}} \FunctionTok{sd}\NormalTok{(results)}
\CommentTok{\# estimated standard error of the mean}
\NormalTok{mean\_se }\OtherTok{\textless{}{-}}\NormalTok{ sample\_sd}\SpecialCharTok{/}\FunctionTok{sqrt}\NormalTok{(n)}

\CommentTok{\# compute the t{-}statistic:}
\NormalTok{t }\OtherTok{\textless{}{-}}\NormalTok{ (sample\_mean }\SpecialCharTok{{-}} \FloatTok{3.5}\NormalTok{) }\SpecialCharTok{/}\NormalTok{ mean\_se}
\NormalTok{t}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 0.3455
\end{verbatim}

At this point you might wonder what the use of \texttt{t} is if the underlying data is not drawn from a normal distribution. In other words: what is the use of \texttt{t} if we cannot interpret it as a value that tells us how likely it is to observe this sample mean, given our null hypothesis? Well, actually we can. And here is where the magic of the CLT comes in: It turns out that there is a mathematical proof (i.e.~the CLT) which states that the t-statistic itself can arbitrarily well be approximated by the standard normal distribution. This is true independent of the distribution of the underlying data in our sample! That is, if we have a large enough sample, we can simply compute the t-statistic and look up how likely it is to observe a value at least as large as t, given the null hypothesis is true (-\textgreater{} the p-value):

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# calculate the p{-}value associated with the t{-}value calculated above}
\DecValTok{2}\SpecialCharTok{*}\FunctionTok{pnorm}\NormalTok{(}\SpecialCharTok{{-}}\FunctionTok{abs}\NormalTok{(t))}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
## [1] 0.7298
\end{verbatim}

In that case we could not reject the null hypothesis of a fair die. However, as pointed out above, the t-statistic is only asymptotically (meaning with very large samples) normally distributed. We might not want to trust this hypothesis test too much because we were using a sample of only 100 observations.

Let's turn back to R in order to illustrate the CLT at work. Similar to the illustration of the LLN above, we will repeatedly compute the t-statistic of our dice play experiment and for each trial increase the number of observations.

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# define the set of sample sizes}
\NormalTok{ns }\OtherTok{\textless{}{-}} \FunctionTok{c}\NormalTok{(}\DecValTok{10}\NormalTok{, }\DecValTok{40}\NormalTok{, }\DecValTok{100}\NormalTok{)}
\CommentTok{\# initiate an empty list to record the results}
\NormalTok{ts }\OtherTok{\textless{}{-}} \FunctionTok{list}\NormalTok{()}
\FunctionTok{length}\NormalTok{(ts) }\OtherTok{\textless{}{-}} \FunctionTok{length}\NormalTok{(ns)}
\CommentTok{\# iterate through each sample size: \textquotesingle{}repeat the die experiment for each sample size\textquotesingle{}}
\ControlFlowTok{for}\NormalTok{ (i }\ControlFlowTok{in} \DecValTok{1}\SpecialCharTok{:}\FunctionTok{length}\NormalTok{(ns)) \{}
     
\NormalTok{     samples.i }\OtherTok{\textless{}{-}} \FunctionTok{sapply}\NormalTok{(}\DecValTok{1}\SpecialCharTok{:}\DecValTok{100000}\NormalTok{, }\ControlFlowTok{function}\NormalTok{(j) }\FunctionTok{sample}\NormalTok{( }\AttributeTok{x =}\NormalTok{ dvalues,}
                                                       \AttributeTok{size =}\NormalTok{ ns[i],}
                                                       \AttributeTok{replace =} \ConstantTok{TRUE}\NormalTok{))}
\NormalTok{     ts[[i]] }\OtherTok{\textless{}{-}} \FunctionTok{apply}\NormalTok{(samples.i, }\ControlFlowTok{function}\NormalTok{(x) (}\FunctionTok{mean}\NormalTok{(x) }\SpecialCharTok{{-}} \FloatTok{3.5}\NormalTok{) }\SpecialCharTok{/} \FunctionTok{sd}\NormalTok{(x), }\AttributeTok{MARGIN =} \DecValTok{2}\NormalTok{)}
\NormalTok{\}}

\CommentTok{\# visualize the result: plot the density for each sample size}

\CommentTok{\# plot the density for each set of t values}
\FunctionTok{hist}\NormalTok{(ts[[}\DecValTok{1}\NormalTok{]], }\AttributeTok{main =} \StringTok{"Sample size: 10"}\NormalTok{, }\AttributeTok{xlab =} \StringTok{"T{-}value"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/clt-1.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{hist}\NormalTok{(ts[[}\DecValTok{2}\NormalTok{]], }\AttributeTok{main =} \StringTok{"Sample size: 40"}\NormalTok{, }\AttributeTok{xlab =} \StringTok{"T{-}value"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/clt-2.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\FunctionTok{hist}\NormalTok{(ts[[}\DecValTok{3}\NormalTok{]], }\AttributeTok{main =} \StringTok{"Sample size: 100"}\NormalTok{, }\AttributeTok{xlab =} \StringTok{"T{-}value"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/clt-3.pdf}

\begin{Shaded}
\begin{Highlighting}[]
\CommentTok{\# finally have a look at the actual standard normal distribution as a reference point}
\FunctionTok{plot}\NormalTok{(}\ControlFlowTok{function}\NormalTok{(t)}\FunctionTok{dnorm}\NormalTok{(t), }\SpecialCharTok{{-}}\DecValTok{4}\NormalTok{, }\DecValTok{4}\NormalTok{, }\AttributeTok{main =} \StringTok{"Normal density"}\NormalTok{)}
\end{Highlighting}
\end{Shaded}

\includegraphics{datahandling_files/figure-latex/clt-4.pdf}

Note how the histogram is getting closer to a normal distribution with increasing sample size.

  \bibliography{references/datahandling.bib,references/packages.bib}

\backmatter
\printindex

\end{document}