\documentclass[a4paper,12pt]{article}
\usepackage[utf8]{inputenc}


\usepackage[english]{babel}
\usepackage{tabularx}
\usepackage{hhline}
%\usepackage[leqno,intlimits]{amsmath}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{epsfig}
%\usepackage{subfigure}
\usepackage{psfrag}
\usepackage{enumerate}
\usepackage{mdwlist}
%\usepackage{draftcopy}

\def\diff{\,\mathrm{d}}
\def\prob{\mathbb{P}}
\def\expect{\mathbb{E}}

\def\IR{\mathbb{R}}
\def\IN{\mathbb{N}}

\def\sigalg{\mathcal{F}}
\def\borel{\mathcal{B}}
\def\leb{\mathrm{Leb}}

\newtheorem{theorem}{Theorem}[section]
\newtheorem{notation}[theorem]{Notation}
\newtheorem{example}[theorem]{Example}
\newtheorem{condition}[theorem]{Condition}
\newtheorem{definition}[theorem]{Definition}
\newtheorem{ansatz}[theorem]{Ansatz}
\newtheorem{remark}[theorem]{Remark}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{sublemma}[theorem]{Sublemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\newtheorem{assumption}[theorem]{Assumption}
\newtheorem{incom}{\textbf{!!!INNER COMMENT!!!}}


%opening
\title{Measure theoretic basics of Probability}
\author{Imre Péter Tóth}

\begin{document}

\maketitle

\begin{abstract}
For the ``Probability 1'' course at CEU, Fall 2013. Occasionally changing draft, with many typos, mistakes, errors and inconsistencies. These notes are mostly copied from my earlier lecture notes on Mathematical Statistical Physics.
\end{abstract}

\tableofcontents

\section*{Introduction}
Nothing of the material presented is my own result. In fact, it is by now considered ``classical'' or ``standard knowledge'' in Mathematics, and I will not attempt to give references to the original papers of the true authors. Instead, I will (when I get to it) cite some textbooks and lecture notes where the material is easily accessible, and which I myself use to reduce the number of false statements written.


\section{Probability warming-up}

\subsection{Measure, measure space, probability, probability space}

\begin{definition}[$\sigma$-algebra]
   For a nonempty set $\Omega$, a family $\sigalg$ of subsets of $\omega$ (i.e. $\sigalg\subset 2^\Omega$, where $2^\Omega:=\{A:A\subset\Omega\}$ is the power set of $\Omega$) is called a $\sigma$-algebra over $\Omega$ if
    \begin{itemize}
     \item $\emptyset\in\sigalg$
     \item if $A\in \sigalg$, then $A^C:=\Omega\setminus A\in \sigalg$ (that is, $\sigalg$ is closed under complement taking)
     \item if $A_1,A_2,\dots\in\sigalg$, then $(\cup_{i=1}^\infty A_i)\in \sigalg$ (that is, $\sigalg$ is closed under countable union).
    \end{itemize}
  \end{definition}

\begin{definition}[measurable space, measurable set]
 If $\Omega$ is a nonempty set and $\sigalg$ is a $\sigma$-algebra over $\Omega$, then the pair $(\Omega,\sigalg)$ is called a measurable space. The elements of $\sigalg$ arer called measurable subsets of $\Omega$.
\end{definition}

\begin{lemma}
A $\sigma$-algerba is closed under finite intersection, countable union and finite union.
\end{lemma}

\begin{proof}
 Homework.
\end{proof}


Note that a $\sigma$-algebra is in general not closed under arbitrary intersection and union. For example, the Borel $\sigma$-algebra on the set $\IR$ of real numbers (see later) contains every $1$-element subset of $\IR$, but it does not contain every subset (a fact we will not prove).

Two trivial examples of $\sigma$-algebra:

\begin{definition}[indiscrete $\sigma$-algebra]
 For a nonempty set $\Omega$, the family of subsets $\sigalg_\text{ind}=\{\emptyset,\Omega\}$ is called the indiscrete or trivial $\sigma$-algebra over $\Omega$.
\end{definition}

\begin{definition}[discrete $\sigma$-algebra]
 For a nonempty set $\Omega$, the family of subsets $\sigalg_\text{discr}=2^\Omega$ (the entire power set) is called the discrete $\sigma$-algebra over $\Omega$.
\end{definition}

It is immediate from the definition that these are indeed $\sigma$-algebras over $\Omega$.

\begin{lemma}
The intersection of any (nonempty) family of $\sigma$-algebras over the same $\Omega$ is also a $\sigma$-algerba over $\Omega$. That is, if $\Omega$ is a nonempty set and $\sigalg_i$ is a $\sigma$-algerba over $\Omega$ for every $i\in I$ where $I$ is \emph{any} nonempty index set, then $\sigalg:=\cap_{i\in I}\sigalg_i$ is also a $\sigma$-algebra over $\Omega$.
\end{lemma}

\begin{proof} (trivial set algebra)
 By definition, $\emptyset\in\sigalg_i$ for every $i\in I$, and $I$ is nonempty, so $\emptyset\in\cap_{i\in I}\sigalg_i=\sigalg$. Similarly, if $A\in\sigalg=\cap_{i\in I}\sigalg_i$, then $A\in\sigalg_i$ for every $i\in I$, so by definition $\Omega\setminus A\in\sigalg_i$ for every $i\in I$, so $\Omega\setminus A\in\sigalg$. Finally, if $A_1,A_2,\dots\in\sigalg$, then $A_1,A_2,\dots\in\sigalg_i$ for every $i\in I$, so by definition $(\cup_{k=1}^\infty A_k)\in \sigalg_i$ for every $i\in I$, which means that $(\cup_{k=1}^\infty A_k)\in \sigalg$.
\end{proof}


It is important to note that $I$ being \emph{any} nonempty set means in particular that it can well be a large set, having infinitely many, or even uncountably many, or possibly much more elements.

\begin{corollary}
If $\Omega$ is a nonempty set and $H\subset 2^\Omega$ is any family of subsets, then there exists a unique $\sigma$-algebra $\sigma(H)$ over $\Omega$, which is the smallest $\sigma$-algebra containig $H$ in the following sense:
 \begin{itemize}
  \item $H\subset\sigma(H)$
  \item If $\sigalg$ is any $\sigma$-algebra over $\Omega$ with $H\subset\sigalg$, then $\sigalg\subset\sigma(H)$.
 \end{itemize}
\end{corollary}

\begin{proof}
The family
$$\{\sigalg\,:\, \text{ $\sigalg$ is a $\sigma$-algebra over $\Omega$ and $H\subset\sigalg$}\}$$
is nonempty, since it contains at least the discrete $\sigma$-algebra $2^\Omega$. Thus by the above lemma,
$$\sigma(H):=\cap\{\sigalg\,:\, \text{ $\sigalg$ is a $\sigma$-algebra over $\Omega$ and $H\subset\sigalg$}\}$$
will do. Uniqueness also follows from the lemma: if there were two diferent such minimal $\sigma$-algebras, their intersection would also be a $\sigma$-algebra, but it would not contain them -- a contradiction.
\end{proof}

\begin{definition}[$\sigma$-algebra generated by a family of sets]
 The above $\sigma(H)$ is called the $\sigma$-algebra generated by $H$.
\end{definition}

\begin{definition}[Borel $\sigma$-algebra]
If $(\Omega,\mathcal{T})$ is a topological space (which means that it makes sense to talk about open subsets of $\Omega$, and $\mathcal{T}$ is the set of these open subsets), then $\borel(\Omega):=\sigma(\mathcal{T})$ is called the Borel $\sigma$-algebra on $\Omega$.
\end{definition}

\begin{remark}
 The set $\mathcal{T}$ is called the \emph{topology}, so the Borel $\sigma$-algebra is the $\sigma$-algebra generated by the topology. For those, who haven't heard, but are interested: A collection $\mathcal{T}\subset 2^\Omega$ of subsets of $\Omega$ is called (by definition) a \emph{topology} over $\Omega$ if it contains $\emptyset$ and $\Omega$, and it is closed under finite intersection and \emph{arbitrary} union. Then the elements of $\mathcal{T}$ are called open sets, and the pair $(\Omega,\mathcal{T})$ is called a topological space. (So the definition only says that $\emptyset$ and $\Omega$ are open, the intersection of finitely many open set is open, and that the union of \emph{any} family of open sets is open.)
\end{remark}

When we talk about the Borel sets on $\IR$ or $\IR^n$, we always think of the usual notion of open sets on these spaces.

\begin{remark}
 Not every subset of $[0,1]$ is Borel. In fact, a non-Borel subset can be constructed (and not only the existence can be proven). We don't go into that.
\end{remark}

\begin{notation}
 We denote by $\IR^+$ the set of nonnegative real numbers -- that is, $\IR^+=[0,\infty)$. In particular, $\IR^+$ includes zero.
\end{notation}


\begin{definition}[measure space, measure]
Let $(\Omega, \sigalg)$ be a measurable space. The nonnegative extended real valued function $\mu$ on $\sigalg$ (that is, $\mu:\sigalg \to \IR^+ \cup \{\infty\}$) is called a measure on $\Omega$, if
\begin{itemize}
 \item $\mu(\emptyset)=0$,
 \item $\mu$ is $\sigma$-additive, meaning that if $\{A_i\}_{i\in I}$ is a countable family of pairwise disjoint measureable sets (with formulas: $A_i\in \sigalg$ for every $i\in I$ where $I$ is a \emph{countable} index set, and $A_i\cap A_j=\emptyset$ for every $i\neq j$, $i,j\in I$), then
$$
\mu\left(\cup_{i\in I} A_i\right)=\sum_{i\in I}\mu(A_i).
$$
\end{itemize}
Then the triple $(\Omega,\sigalg,\mu)$ is called a measure space and $\mu(A)$ is called the measure of the set $A$.
\end{definition}

\begin{remark}
It is absolutely important that in the definition of $\sigma$-additivity $I$ is countably infinite: $\sigma$-additivity is more than finite additivity and less than arbitrary additivity. For example, if $\mu$ is the Lebesgue measure on $\IR$ (see later), every $1$-element set $\{x\}$ has $\mu(\{x\})=0$, which implies that every countable set has zero measure, but of course
$$1=\mu([0,1])=\mu(\cap_{x\in[0,1]} \{x\})\neq \sum_{x\in [0,1]} \mu(\{0\})=\sum_{x\in [0,1]} 0=0.$$
(Whatever the sum of uncountably many real numbers could mean.)
\end{remark}

Once it came up, we mention the following, \emph{absolutely non-important} definition:

\begin{definition}[sum of many nonnegative extended real numbers]
 If $a_i\in \IR^+\cup\{\infty\}$ for every $i\in I$ where $I$ is \emph{any} index set, then we define the sum of all $a_i$ as
$$
\sum_{i\in I}a_i:=\sup\left\{ \sum_{i\in J} a_i\,:\,  \text{ $J\subset I$ and $J$ is finite} \right\}.
$$
\end{definition}

Note that it is important that the $a_i$ are nonnegative. This definition obviously coincides with the usual sum of (an arbitrarily reorderable) series if $I$ is countable. This new notion of an infinite sum is no serious extention of the well known notion of a countable series: it is easy to see that if the sum is finite, then at most countably many terms can be nonzero. 

\begin{remark}
In the definition of the measure, the first requirement $\mu(\emptyset)=0$ is almost automatic from $\sigma$-additivity: it's only there to rule out the trivial nonsense $\mu(\emptyset)=\infty$. In fact it would be enough to require that at least one measurable set $A$ has finite measure: $\sigma$-additivity implies
$$
\mu(A)=\mu(A\cup\emptyset\cup\emptyset\cup\emptyset\cup\dots)=\mu(A)+\mu(\emptyset)+\mu(\emptyset)+\mu(\emptyset)+\dots=\mu(A)+\sum_{i=1}^\infty \mu(\emptyset).
$$
If $\mu(A)<\infty$, then this implies $0=\sum_{i=1}^\infty \mu(\emptyset)$, so $\mu(\emptyset)=0$.
\end{remark}

\begin{definition}
The measure $\chi$ on a nonempty set $\Omega$ equipped with the discrete $\sigma$-algebra $2^\Omega$ defined as
$$
\chi(A):=\sharp A:=\begin{cases}
                   \text{number of elements in $A$,} & \text{if $A$ is finite}\\
                   \infty, & \text{if $A$ is infinite}
                  \end{cases}
$$
is called the counting measure on $\Omega$. The restriction of $\chi$ to any $\sigma$-algebra $\sigalg\subset 2^\Omega$ is still called a counting measure (on $\sigalg$).
\end{definition}

One of the most important examples of a measure is the Lebesgue measure on $\IR$ or on $\IR^d$

\begin{definition}[Lebesgue measure vaguely]
Consider the set $\IR$ with the Borel $\sigma$-algebra $\borel$. The measure $\leb:\borel\to\IR^+\cup\{\infty\}$ is called the Lebesgue measure on $\IR$, if it assigns to every interval its length -- that is, for every $a,b\in\IR$, $a\le b$ we have
$$
\leb((a,b))=\leb((a,b])=\leb([a,b))=\leb([a,b])=b-a.
$$
The restriction of $\leb$ to a Borel subset of $\IR$ (e.g. an interval $[c,d]$ or $(c,\infty)$) is still called Lebesgue measure and is still denoted by $\leb$. (More precisely, if $(\IR,\borel,\leb)$ is the Lebesgue measure space on $\IR$, and $I\in\borel$, than one can define the ``restriction of $\leb$ to $I$'' as the measure space $(I,\borel_I,\leb_I)$ where $\borel_I:=\{A\cap I\,:\, A\in\borel\}=\{B\,:\, B\in\borel, B\subset I\}\subset\borel$ and $\leb_I:=\leb\arrowvert_{\borel_I}$ is the restriction of $\leb$ to $\borel_I$.)

Similarly, the ``Lebesgue measure on $\IR^d$'' is the measure on Borel subsets of $\IR^d$ which assigns to every box its $d$-dimensional volume, i.e. for every $a_1\le b_1,a_2\le b_2,\dots a_d\le b_d\in\IR$ we have  
$$
\leb_d([a_1,b_1]\times[a_2,b_2]\times\dots\times[a_d,b_d])=\prod_{i=1}^d (b_i-a_i).
$$
Restrictions to Borel subsets of $\IR^d$ are still called Lebesgue measure, and denoted by $\leb_d$ or just $\leb$.
\end{definition}

\begin{remark}
 The above ``definition'' of the Lebesgue measure is far from being complete, and is not the usual definition -- it's actually a characterization of the Lebesgue measure which shows its essence. It can be (and needs to be) shown that such a measure indeed exists, since in the ``definition'' we only gave the value of $\leb$ for a few very special sets, and not every Borel set. Also uniqueness can and needs to be shown. These questions lead to the construction of measures based on their pre-known values on certain pre-chosen ``to-be-measurable'' sets, which can sometimes be of crucial importance, but we don't go into that here.
\end{remark}

\begin{remark}
 In the measure theory literature, Lebesgue measure is defined on a $\sigma$-algerba $\sigalg$ which is \emph{larger} than the Borel $\sigma$-algebra (i.e. $\borel\subsetneqq\sigalg$), called the ``set of Lebesgue measurable sets''. In particular, $\sigalg$ has the property that if $B\in\sigalg$, $\leb(B)=0$ and $A\subset B$, then $A\in\sigalg$, which is not true for Borel sets. However, in probability theory it is usual to consider Lebesgue measure restricted to Borel sets only (as in the above definition).
\end{remark}

The following definition shows that the basic object of probability theory, called ``the probability'' is in fact a measure.

\begin{definition}[Kolmogorov probability space]
The triple $(\Omega,\sigalg,\prob)$ is called a Kolmogorov probability space (or just probability space) if it is a measure space and $\prob(\Omega)=1$.
Then $\prob$ is called the \emph{probability} or a ``probability measure'', elements of $\sigalg$ are called \emph{events}, and elements of $\Omega$ are the \emph{elementary events}. For $A\in\sigalg$, $\prob(A)\in [0,1]$ is called the probability of the event $A$.
\end{definition}

Picture $\omega\in\Omega$ as possible outcomes of an experiment, so an event $A\in\sigalg$ often consists of many possible outcomes of that experiment, which have some common property that we are interested in. By definition, an ``event'' is something which has a probability.

\subsection{Measurable functions, random variables and their distributions}

\begin{notation}
For a function $f:\Omega\to\Omega'$ and a set $A'\subset\Omega'$, let $f^{-1}(A')$ denote, as usual, the complete inverse image of $A'$ defined as $f^{-1}(A'):=\{\omega\in\Omega\,:\,f(\omega)\in A'\}$. Note that this makes sense for any function and any $A'$ -- in particular, $f$ need not be invertable.
\end{notation}


\begin{definition}[measureable function, observable, random variable]
Let $(\Omega,\sigalg)$ and $(\Omega',\sigalg')$ be measurable spaces. The function $X:\Omega\to\Omega'$ is called \emph{measurable} or $(\sigalg,\sigalg')$-measurable, if for every $A'\in\sigalg'$ we have $X^{-1}(A')\in\sigalg$. (That is, if the inverse image of any measurable set is also measurable.) In physical applications, when $\Omega$ is the (possibly complicated) phase space of a system and $\Omega'$ is a (usually simple) set of possible measurement results (e.g. $\Omega'=\IR$), the same $X$ is called an \emph{observable}. In the context of probability theory, when $(\Omega,\sigalg,\prob)$ is a probability space, $X$ is called a ($\Omega'$-valued) \emph{random variable}.
\end{definition}


Note that the notion of measurability of a function depends on the choice of the $\sigma$-algebras $\sigalg$ and $\sigalg'$. However, in many cases when this choice is clear from the context, we simply say ``measurable'' instead of ``$(\sigalg,\sigalg')$-measurable''.
When we talk about a random variable, and do not specify the range, usually $(\Omega',\sigalg')=(\IR,\borel)$ is understood.

\begin{remark}
 If $X:\Omega\to\Omega'$ is a random variable and $x\in\Omega'$, then we denote by $\{X=x\}$ the set of elementary events where $X$ takes the value $x$ -- that is,
$$
\{X=x\}:=\{\omega\in\Omega\,:\, X(\omega)=x\}=X^{-1}(\{x\}).
$$
Similarly, if $A'\subset\Omega'$, then $\{X\in A'\}$ denotes the set of elementary events where $X$ takes values in $A'$:
$$
\{X\in A'\}:=\{\omega\in\Omega\,:\, X(\omega)\in A'\}=X^{-1}(A).
$$
 With this in mind, the definition of a random variable as a \emph{measurable} function is very natural. The definition says exactly that is $A'$ is a measurable subset of the range $\Omega'$, then the set $\{X\in A\}$ is indeed an \emph{event} -- that is, it makes sense to talk about its probability.
\end{remark}



\begin{example}
 coordinate, number rolled, sum of these
\end{example}


Random variables are the central objects of study in probability theory. In a typical situation they extract fairly little information (e.g. a single number) from a big probability space containing many complicated possible outcomes of an experiment. So to ``understand'' a random variable $X:\Omega\to\Omega'$ on the probability space $(\Omega,\sigalg,\prob)$ well, we need less information than the what $\prob$ and $X$ contain. From another point of view: when we consider a random variable, $\Omega$ is often not needed, or not even known. All we need to know is the possible values (in $\Omega'$) $X$ can take, and the probability of these being taken. This information is contained exactly in a measure on $\Omega'$, as the following definition shows.

\begin{definition}[distribution of a random variable]
Let $(\Omega,\sigalg,\prob)$ be a probability space, $(\Omega',\sigalg')$ a measurable space and $X:\Omega\to\Omega'$ a random variable. Then the distribution of $X$ is the measure $\mu$ on $(\Omega',\sigalg')$ which is defined as
$$
\mu(A):=\prob(\{X\in A\})=\prob(X^{-1}(A)) \quad \text{for every $A\in\sigalg'$}.
$$
This can be written in short as
$$
\mu:=\prob\circ X^{-1}.
$$
$\mu$ is nothing else than the ``push-forward'' of the probability $\prob$ by $X$ to $\Omega'$.
\end{definition}

In the special case when $\Omega'=\IR$ and $X$ is discrete (meaning that it can take finitely many, or at most countably many values), there is a convenient alternative way to ``desribe the distribution of $X$'', by simply listing the possible values $x_k$ and their probabilities $p_k:=\prob(X=x_k):=\prob(\{X=x_k\})$. Then the information contained in the sequence of pairs $\{(x_k,p_k)\}_{k=1}^N$ (with possibly $N=\infty$) is called the \emph{discrete probability distribution}. Having this information, one can calculate probabilities of events by summation:
$$
\mu(A)=\prob(\{X\in A\})=\sum_{k:x_k\in A}p_k.
$$

Similarly, in the special case when $\Omega'=\IR$ and $X$ is absolutely continuous (see later), there is convenient alternative ``description of the distribution'' by a density function $f:\IR\to\IR^+$ from which one can calculate probabilities of events by integration:
$$
\mu(A)=\prob(\{X\in A\})=\int_A f(x)\diff x.
$$

The above notion of a probability distribution is a far-reaching generalization of both notions.

\begin{example}
 sum of two rolled numbers
 real number generated by a sequence of fair coin tosses
 real number generated by a sequence of biased coin tosses
\end{example}

\subsection{Integral and expectation}

\subsubsection{Integral, integrability}

For an (extended) \emph{real-valued} measurable functions $X:\Omega\to\IR$ on a measure space $(\Omega,\sigalg,\mu)$ it makes sense to talk about the integral $\int_\Omega X\diff \mu$. This is an essential tool, and also an important object of study both in measure theory and in probability theory. We don't go deep into the definition and properties of the integral here -- we don't want to, and we can't substitute a measure theory course now. I just give very briefly one of the shortest possible definitions, and point out a few main feaures.

Since we don't want to exclude the case when either a function or a measure takes the value $\infty$, we work with extended real numbers, and use the convention
$$
0\cdot\infty:=\infty\cdot 0:=0.
$$

We start by defining the integral of nonnegative functions.

\begin{definition}[integral of non-negative extended real valued functions]
If $(\Omega,\sigalg,\mu)$ is a measure space and $X:\Omega\to\IR^+\cup\{\infty\}$ is measurable, we introduce a sequence $X_n:\Omega\to\IR^+$ of \emph{simple} functions (i.e. taking only finitely many values) which approximate $X$ (from below) as
$$
X_n(\omega):=\max\{x\in\{0,\frac1{2^n},\frac2{2^n},\dots,2^n-\frac1{2^n},2^n\}\,:\,x\le X(\omega)\}.
$$
Than we define the $n$-th integral-approximating sum as
$$
I_n:=\sum_{x\in\{0,\frac1{2^n},\frac2{2^n},\dots,2^n-\frac1{2^n},2^n\}} x\mu(X_n^{-1}(\{x\})),
$$
and the integral of $X$ as
$$
\int_\Omega X\diff \mu:=\lim_{n\to\infty} I_n.
$$
\end{definition}

The sets $X_n^{-1}(\{x\})\subset\Omega$ in the definition of $I_n$ are ensured to be $\sigalg$-measurable by the assumption that $X$ is measurable (and the fact that $\{x\}$ is Borel-measurable in $\IR$), so $\mu(X_n^{-1}(\{x\}))$ makes sense. The sequence $X_n$ of functions is cleverly defined to be increasing, and so is the sequence $I_n$, so the limit in the above definition exists, but is possibly infinite.

We can now go on to the general definition of the integral for extended real valued functions:

\begin{definition}[integral]
If $(\Omega,\sigalg,\mu)$ is a measure space and $X:\Omega\to\IR \cup\{-\infty,\infty\}$ is measurable,
we introduce the positive part $X_+$ and the negative part $X_-$ of $X$ as
$$
X_+(\omega):=\begin{cases}
              X(\omega), & \text{if $X(\omega)>0$,}\\
              0, & \text{if not} 
             \end{cases},
\quad
X_-(\omega):=\begin{cases}
              -X(\omega), & \text{if $X(\omega)<0$,}\\
              0, & \text{if not} 
             \end{cases}.
$$
Note that both $X_+$ and $X_-$ are nonnegative and $X=X_+-X_-$.

Now
\begin{itemize}
 \item If either $\int_\Omega X_+ \diff \mu<\infty$ or $\int_\Omega X_- \diff \mu<\infty$, then we define the integral of $X$ as
$$
\int_\Omega X \diff \mu:=\int_\Omega X_+ \diff \mu-\int_\Omega X_- \diff \mu,
$$ 
which can possibly be $\infty$ or $-\infty$. 
 \item If both $\int_\Omega X_+ \diff \mu=\infty$ and $\int_\Omega X_- \diff \mu=\infty$, then we say that the integral of $X$ doesn't exist (or that it is undefined).
\end{itemize}
\end{definition}

In the usual mathematical language, there is an important distinction between the existence of an integral and the integrability of a function. We emphasize this in the following definition:

\begin{definition}[integrability]
Let $(\Omega,\sigalg,\mu)$ be a measure space and $X:\Omega\to\IR \cup\{-\infty,\infty\}$ measurable. If the integral $\int_\Omega X\diff\mu$ exists and it is \emph{finite}, then we say that $X$ is integrable (with respect to $\mu$).
\end{definition}

So integrability of $X$ means $-\infty < \int_\Omega X \diff \mu<\infty$, which is equivalent to both $\int_\Omega X_+ \diff \mu$ and $\int_\Omega X_- \diff \mu$ being finite.

\begin{remark}[comparison to the Riemann integral]
The above definition of the integral is similar to the construction of the good old Riemann integral: in both cases the domain of integration, $\Omega$, is chopped up into pieces, on each of which the function $X$ takes nearly constant values. Than the ``size'' of each piece is multiplied by the approximate value of the function there, and these products are added up to obtain an integral approximating sum. The crucial difference is that in the case of Riemann integral, these small pieces of $\Omega$ had to be \emph{intervals}, while here they can be any measurable subset of $\Omega$. In particular, the points in $X_n^{-1}(\{x\})$ don't need to be ``close'' to each other in any sense, thus 
$\Omega$ absolutely doesn't need to be the real line or anything similar. It doesn't need to have any additional structure that would give sense to the words ``distance'' or ``being close''. Really, any measure space will do.
\end{remark}

A trivial but important example of integrable functions:

\begin{lemma}[bounded functions on finite measure spaces are integrable]
If $(\Omega,\sigalg,\mu)$ is a finite measure space (meaning that $\mu(\Omega)<\infty$) and $X:\Omega\to\IR$ is measurable and bounded -- meaning that there exists an $M\in\IR$ such that $-M\le X(\omega)\le M$ for every $\omega\in\Omega$, then $X$ is integrable w.r.t. $\mu$. 
\end{lemma}

\begin{proof}
 $X$ being bounded by $M$ implies that $0\le X_+(\omega),X_-(\omega)\le M$. when calculating the integral of, say, $X_+$, we have $X_n(\omega)\le X_+(\omega)\le M$, and thus
\begin{eqnarray*}
I_n&:=&\sum_{x\in\{0,\frac1{2^n},\frac2{2^n},\dots,2^n-\frac1{2^n},2^n\}} x\mu(X_n^{-1}(\{x\}))\\
&\le& M \sum_{x\in\{0,\frac1{2^n},\frac2{2^n},\dots,2^n-\frac1{2^n},2^n\}} \mu(X_n^{-1}(\{x\}))\\
&=&M \mu(\cup_{x\in\{0,\frac1{2^n},\frac2{2^n},\dots,2^n-\frac1{2^n},2^n\}} X_n^{-1}(\{x\}))\\
&=&M \mu(\Omega).
\end{eqnarray*}
So we get $\int_\Omega X_+\diff\mu<M\mu(\Omega)$. Similarly, $\int_\Omega X_-\diff\mu<M\mu(\Omega)$,
so
$$
-\infty<-M\mu(\Omega)\le \int_\Omega X\diff\mu\le M\mu(\Omega)<\infty.
$$
\end{proof}

A basic property of the integral with a huge importance is linearity in the function integrated (the integrand):

\begin{theorem}[linearity of integrability and the integral]\label{thm:int_linear}
Let $X_1$ and $X_2$ be real valued measurable functions on the same probability space $(\Omega,\sigalg,\mu)$, and let $\alpha_1,\alpha_2\in\IR$. If both $I_1:=\int_\Omega X_1\diff \mu$ and $I_2:=\int_\Omega X_2\diff \mu$ exist and $\alpha_1 I_1+\alpha_2 I_2$ is not of the form $\infty-\infty$, then $\int_\Omega (\alpha_1 X_1+\alpha_2 X_2)\diff \mu$ exists and
$$\int_\Omega (\alpha_1 X_1+\alpha_2 X_2)\diff\mu=\alpha_1 \int_\Omega X_1\diff \mu+\alpha_2 \int_\Omega X_2\diff \mu.$$
As a consequence, if $X_1$ and $X_2$ are both integrable, then so is $\alpha_1 X_1+\alpha_2 X_2$.
\end{theorem}

The proof is easy from the definition, but we don't discuss it here. See any measure theory book.
It is useful to note that the integral is linear not only in the integral, but also in the measure:

\begin{theorem}[linearity of the integral II.]
Let $(\Omega,\sigalg)$ be a measurable space, $\mu_1$ and $\mu_2$ measures on it, $X$ a real valued measurable function and $0\le \alpha_1,\alpha_2\in\IR$. If both $I_1:=\int_\Omega X\diff \mu_1$ and $I_2:=\int_\Omega X\diff \mu_2$ exist and $\alpha_1 I_1+\alpha_2 I_2$ is not of the form $\infty-\infty$, then $\int_\Omega X \diff(\alpha_1 \mu_1+\alpha_2 \mu_2)$ exists and
$$\int_\Omega X \diff(\alpha_1 \mu_1+\alpha_2 \mu_2)=\alpha_1 \int_\Omega X\diff \mu_1+\alpha_2 \int_\Omega X\diff \mu_2.$$
As a consequence, if $X$ is integrable w.r.t. both $\mu_1$ and $\mu_2$, then so it is w.r.t. $\alpha_1 \mu_1+\alpha_2 \mu_2$.
\end{theorem}

\begin{remark}[bilinearity of the integral]
In the last theorem, we required that $\alpha_1$ and $\alpha_2$ be nonnegative -- otherwise $\alpha_1 \mu_1+\alpha_2 \mu_2$ may not be a measure, since in our definition a measure has to be nonnegative. For the same reason, the measures on a measurable space do not form a linear space. In functional analysis, to overcome that limitation, it is common to introduce the notion of ``signed measures'' (say, as a difference of two measures), which already form a linear space (with the usual notion of addition and multiplication). Then the two-variable real-valued mapping
$$\langle \mu,X\rangle:=\int X\diff\mu$$
can be defined on suitably chosen linear spaces of measures and functions (e.g. $\mu\in\{\text{signed finite measures}\}$, $X\in\{\text{bounded measurable functions}\}$.)
The above two theorems show that this mapping is bilinear, which is the property where functional analysis starts.
\end{remark}

\subsubsection{Exchanging limits with integrals}

It is utmost important that Theorem~\ref{thm:int_linear} is about the linear combination of \emph{two} integrable functions. Of course, it immediately implies linearity of the integral for \emph{finite} linear combinations, but \emph{does not say anything about infinite sums}. Indeed, linearity of the integral for infinite sums is not at all true in general. In fact, it is an important issue, in which cases exchanging an integral with a limit is possible -- one has to be at least always careful. In the following we state (without proof) three theorems, which are the most frequently (and almost exclusively) used tools in checking exchangeability. In a situation where none of them works, exchangeability is usually hard to prove, and may very well not be true.

The first and most used tool is the Lebesgue dominated convergence theorem:

\begin{theorem}[dominated convergence]
Let $(\Omega,\sigalg,\mu)$ be a measure space and $f_1,f_2,\dots$ measurable real valued functions on $\Omega$ which converge to the limit function pointwise, $\mu$-almost everywehere. (That is, $\lim_{n\to\infty}f_n(x)=f(x)$ for every $x\in\Omega$, except possibly for a set of $x$-es with $\mu$-measure zero.)
Assume furthermore that the $f_n$ admit a common \emph{integrable dominating function}: there exists a $g:\Omega\to\IR$ such that
$|f_n(x)|\le g(x)$ for every $x\in\Omega$ and $n\in\IN$, and $\int_\Omega g\diff \mu<\infty$. Then (all the $f_n$ and also $f$ are integrable and)
$$\lim_{n\to\infty} \int_\Omega f_n \diff \mu = \int_\Omega f \diff \mu.$$
\end{theorem}

As you see, it is enough to require ``almost everywhere'' convergence, which is no surprise, since changing a function on a set of measure zero doesn't change the integral. In fact, it would be enough to require that $g$ dominates the $f_n$ almost everywhere -- moreover, it would be enough to require that the $f_n$ and $g$ be extended real valued and defined almost everywhere. This is not a serious generalization, so I decided to rather keep the formulation simple. In the literature usually the most general form is given.

The second and easiest tool is Beppo Levi's monotone convergence theorem:

\begin{theorem}[monotone convergence]
Let $(\Omega,\sigalg,\mu)$ be a measure space and $f_1,f_2,\dots$ a sequence of measurable \emph{nonnegative} real valued functions on $\Omega$ which is pointwise increasing. (That is, $0\le f_n(x)\le f_{n+1}(x)$ for every $n\in\IN$ and $x\in\Omega$.)
Then the pointwise limit function $f$ defined by $f(x):=\lim_{n\to\infty} f_n(x)$ is also measurable and
$$\lim_{n\to\infty} \int_\Omega f_n \diff \mu = \int_\Omega f \diff \mu.$$
\end{theorem}

The third and trickiest tool is the Fatou lemma:

\begin{theorem}[Fatou lemma]
Let $(\Omega,\sigalg,\mu)$ be a measure space and $f_1$, $f_2$, \dots a sequence of measureabale functions $f_n:\Omega\to\IR$, which are nonneagtive, e.g. $f_n(x)\ge 0$ for every $n=1,2,\dots$ and every $x\in\Omega$. Then
$$\int_\Omega \liminf_{n\to\infty} f_n(x) \diff \mu(x)\le \liminf_{n\to\infty} \int_\Omega f_n(x)\diff \mu(x)$$
(and both sides make sense).
\end{theorem}


\begin{definition}[Absolute continuity of measures, singularity of measures]
Let $\mu$ and $\nu$ be two measures on the same measurable space $\Omega,\sigalg$. We say that $\nu$ is absolute continuous with respect to $\mu$ (notation: $\nu\ll \mu$) if for every $A\in\sigalg$ for which $\mu(A)=0$, also $\nu(A)=0$. We say that $\nu$ and $\mu$ are singular (with respect to each other) (notation: $\nu\perp\mu$) if there exists and $A\in\sigalg$ for which $\nu(A)=0$ and $\mu(\Omega\setminus A)=0$.
\end{definition}

The best known probability distributions are all examples of either one or the other of these: the ``discrete'' probability distributions are singular w.r.t. Lebesgue measurer on $\IR$, while the ones that are loosely called ``continuous'' are actually absolutely continuous w.r.t. Lebesgue measurer on $\IR$.

\begin{theorem}[Radon-Nykodim]
If $\mu$ and $\nu$ are two measures on the same measurable space $\Omega,\sigalg$ and $\nu\ll \mu$, then there exists a measurable $f:\Omega\to\IR$, called the \textbf{density} of $\nu$ w.r.t. $\mu$, which satisfies $\nu(A)=\int_A f \diff\mu$. 
\end{theorem}



absolute continuity w.r.t. counting measure

absolute continuity w.r.t. Lebesgue measure

integration w.r.t counting measure

integration w.r.t. Lebesgue measure

integration w.r.t. absolutely continouos measures

density function, distribution function

continuity of measures

expectation if it exists
  - real-valued
  - complex or $R^d$ -valued
  - remark about more complicated spaces

integration by substitution

\begin{theorem}
 Let $(\Omega,\sigalg)$
 and $(\Omega',\sigalg')$
 be measurable spaces, $\mu$ a measure on $\Omega$,
$X:\Omega\to\Omega'$ measurable and $\nu:=\mu\circ X^{-1}$ the push-forward of $\mu$ by $X$ (a measure on $\Omega'$) (the distribution of $X$, if $\mu$ is a probability). Then for any measurable $g:\Omega'\to \IR$
$$\int_\Omega g(X)\diff \mu=\int_{\Omega'}g \diff \nu.$$ 
\end{theorem}


expectation of a distribution

linearity of expectation

variance, standard deviation

moments, centered moments, moment-generating function, characteristic function

product measure space, product measure - be careful with infinite products

pairwise independence, marginal distributions

mutual independence

variance of linear combination of independent random variables

Markov inequality

weak law of large numbers

central limit theorem

weak convergence of measures

weak convergence of random variables

continuity of the characteristic function



\end{document}

