\documentclass[10pt]{article}
\newtheorem{define}{Definition}
%\newcommand{\Z}{{\mathbb{Z}}}
\usepackage{amsfonts}
\usepackage{cleveref}

\oddsidemargin=0.15in
\evensidemargin=0.15in
\topmargin=-.5in
\textheight=9in
\textwidth=6.25in

\begin{document}
\input{preamble.tex}

%{lecture number}{lecture date}{Ronitt Rubinfeld}{Your name}
\lecture{15}{04/04, 2012}{Ronitt Rubinfeld}{Mohsen Ghaffari}

%%%% body goes in here %%%%

\section{Recap}
\begin{definition}Consider function $f:\{\pm 1\}^{n} \rightarrow R$. We say $f$ has $\alpha(\epsilon, n)$-Fourier Concentration if $$\sum_{S \subseteq [n] \,,\, |S| > \alpha(\epsilon, n)} \hat{f}^2(S)\leq \epsilon$$
\end{definition}

\begin{theorem}If $\;\mathcal{C}$ has Fourier Concentration at most $d=\alpha(\epsilon, n)$, then there is an $O(\frac{n^d}{\epsilon})$ sample-uniform learning algorithm for $\mathcal{C}$.
\end{theorem}

\begin{definition}\label{def:ns} Let $N_{\epsilon}(x)$ be what we get from randomly flipping each bit of $x$ with probability $\epsilon$ (iid). Then noise sensitivity of function $f$, $\textit{ns}_{\epsilon}(f)$, is defined as $ \textit{ns}_{\epsilon}(f) =\Pr_{x \in \{\pm1\}^{n}}\big[f(x) \neq f(N_{\epsilon}(x))\big]$.
\end{definition}

\begin{theorem}\label{thm:Peres}For any linear threshold function $f$, we have $\textit{ns}_{\epsilon}(f) \leq 8.8 \sqrt{\epsilon}$.
\end{theorem}

%------------------------------------------------------------------------------------------------------------------------------------------------
%------------------------------------------------------------------------------------------------------------------------------------------------
%------------------------------------------------------------------------------------------------------------------------------------------------
%------------------------------------------------------------------------------------------------------------------------------------------------
\section{Today}
\subsection{Noise Sensitity vs. Fourier Concentration}
\begin{theorem} \label{thm:ns1} Consider function $f:\{\pm 1\}^{n} \rightarrow \{\pm 1\}$. Then, we have $$ \textit{ns}_{\epsilon}(f) = \frac{1}{2} - \frac{1}{2} \sum_{S \subseteq [n]} (1-2\epsilon)^{|S|} \hat{f}^2(S)$$
\end{theorem}
%------------------------------------------------------------------------------------------------------------------------------------------------
\begin{proof}
By Definition \ref{def:ns}, we have
\begin{eqnarray}
\textit{ns}_{\epsilon}(f) = &&\Pr_{x \in \{\pm1\}^{n}}\big[f(x) \neq f(N_{\epsilon}(x))\big] \nonumber\\ 
=&&\Pr_{x \in \{\pm1\}^{n}, y=N_{\epsilon}(x)}\big[f(x) \neq f(y)\big] \nonumber\\
=&&\mathbb{E}_{x, y} \big[ \mathbf{1}_{f(x)\neq f(y)}\big] \nonumber\\
=&&\mathbb{E}_{x, y} \big[ \frac{(f(x)- f(y))^2}{4} \big] \nonumber\\
=&&\mathbb{E}_{x, y} \big[ \frac{2 -2 f(x)f(y)}{4} \big] \nonumber \\
=&&\frac{1}{2} - \frac{1}{2}\mathbb{E}_{x, y} \big[ f(x)f(y) \big] \nonumber \\
=&&\frac{1}{2} - \frac{1}{2}\sum_{s, T \subseteq [n]} \hat{f}(S) \hat{f}(T) \; \mathbb{E}_{x, y} \big[ \chi_{S}(x)\chi_{T}(y) \big] \nonumber \\
=&&\frac{1}{2} - \frac{1}{2}\sum_{s, T \subseteq [n]} \hat{f}^2(S) \; \mathbb{E}_{x, y} \big[ \chi_{S}(x)\chi_{S}(y) \big] 
\label{eq:ns}
\end{eqnarray}
Now, we evaluate $\mathbb{E}_{x, y} \big[ \chi_{S}(x)\chi_{S}(y) \big]$. Let $e_{x_i}$ (resp. $e_{y_i}$) be the unit vector that has value $x_i$ (resp $y_i$) at position $i$ and $1$ at all the other places. Then, we have
\begin{eqnarray}\mathbb{E}_{x, y} \big[ \chi_{S}(x)\chi_{S}(y) \big] 
= && \mathbb{E}_{x, y} \big[ \prod_{i=1}^{n} \chi_{S}(e_{x_i})\chi_{S}(e_{y_i})\big] \nonumber\\
= && \mathbb{E}_{x, y} \big[ \prod_{i\in S} \chi_{S}(e_{x_i})\chi_{S}(e_{y_i})\big] \nonumber\\
= && \prod_{i\in S} \mathbb{E}_{x, y} \big[ \chi_{S}(e_{x_i})\chi_{S}(e_{y_i})\big] \nonumber\\
= && \prod_{i\in S} (\Pr[\chi_{S}(e_{x_i}) = \chi_{S}(e_{y_i})] - \Pr[\chi_{S}(e_{x_i}) \neq \chi_{S}(e_{y_i})])  \nonumber\\
= && \prod_{i\in S} (\Pr[x_i= y_i] - \Pr[x_i\neq y_i])  \nonumber\\
= && \prod_{i\in S} (1-2\epsilon)  \nonumber\\
= && (1-2\epsilon)^{|S|} 
\end{eqnarray}
Hence, substituting this into (\ref{eq:ns}), we get $$ \textit{ns}_{\epsilon}(f) = \frac{1}{2} - \frac{1}{2} \sum_{S \subseteq [n]} (1-2\epsilon)^{|S|} \hat{f}^2(S)$$
\end{proof}
%------------------------------------------------------------------------------------------------------------------------------------------------

Using this result, we can get the following connection between noise sensitivity of a function and its Fourier Concentration.
\begin{theorem}\label{thm:ns2} Consider function $f:\{\pm 1\}^{n} \rightarrow \{\pm 1\}$. Then, for any $0 < \gamma <\frac{1}{2}$, we have 
$$ \sum_{S\subseteq[n], |S| \geq \frac{1}{\gamma}}{\hat{f}^2(S)} \leq 2.32\; \textit{ns}_{\gamma}(f)$$
\end{theorem}
%------------------------------------------------------------------------------------------------------------------------------------------------
\begin{proof}Using \Cref{thm:ns1}, we have
\begin{eqnarray} 2\; \textit{ns}_{\gamma}(f)  
&& = 1 -  \sum_{S \subseteq [n]} (1-2\gamma)^{|S|} \hat{f}^2(S) \nonumber \\
&& = \sum_{S \subseteq [n]} \hat{f}^2(S) -  \sum_{S \subseteq [n]} (1-2\gamma)^{|S|} \hat{f}^2(S) \nonumber \\
&& = \sum_{S \subseteq [n]} \big(1 -  (1-2\gamma)^{|S|}\big) \; \hat{f}^2(S) \nonumber \\
&& \geq \sum_{S \subseteq [n], |S| \geq \frac{1}{\gamma} } \big(1 -  (1-2\gamma)^{|S|}\big) \; \hat{f}^2(S) \nonumber \\
&& \geq \sum_{S \subseteq [n], |S| \geq \frac{1}{\gamma} } \big(1 -  (1-2\gamma)^{\frac{1}{\gamma}}\big) \; \hat{f}^2(S) \nonumber \\
&& \geq \sum_{S \subseteq [n], |S| \geq \frac{1}{\gamma} } \big(1 -  e^{-2} \big) \; \hat{f}^2(S) \nonumber \\
\end{eqnarray}
Hence, $$ \sum_{S\subseteq[n], |S| \geq \frac{1}{\gamma}}{\hat{f}^2(S)} \leq \frac{2}{1-e^{-2}} \; \textit{ns}_{\gamma}(f) \leq 2.32\; \textit{ns}_{\gamma}(f)  $$
\end{proof}
%------------------------------------------------------------------------------------------------------------------------------------------------
\subsection{Application: Learning Half-Space Functions}
The above result gives us a couple of interesting corollaries as follows.
\begin{corollary}\label{crl:ns1}Consider function $f:\{\pm 1\}^{n} \rightarrow \{\pm 1\}$. Also define function $\beta:[0, 0.5] \rightarrow [0, 0.5]$ such that $\textit{ns}_{\gamma}(f) \leq \beta(\gamma) $ for every $\gamma \in [0, 0.5]$. Then, we have 
$$ \sum_{S\subseteq[n], |S| \geq \frac{1}{\beta^{-1}(\frac{\epsilon}{2.32})}}{\hat{f}^2(S)} \leq \epsilon$$
\end{corollary}
%------------------------------------------------------------------------------------------------------------------------------------------------
\begin{proof}By \Cref{thm:ns2}, we have $$ \sum_{S\subseteq[n], |S| \geq \frac{1}{\beta^{-1}(\frac{\epsilon}{2.32})}}{\hat{f}^2(S)} \leq  2.32\;\; \textit{ns}_{\beta^{-1}(\frac{\epsilon}{2.32})}(f) \leq 2.32 \;\;\beta(\beta^{-1}(\frac{\epsilon}{2.32})) = \epsilon$$
\end{proof}
%------------------------------------------------------------------------------------------------------------------------------------------------
\begin{corollary}\label{crl:ns2}Consider half-space function $h:\{\pm 1\}^{n} \rightarrow \{\pm 1\}$. Then, we have 
$$ \sum_{S\subseteq[n], |S| \geq \Theta(\frac{1}{\epsilon^2})}{\hat{f}^2(S)} \leq \epsilon$$
\end{corollary}
\begin{proof} From previous lecture, we know that  for a half-space function $h$, we have $\textit{ns}_{\epsilon}(h) \leq 8.8 \sqrt{\epsilon}$. Let $\beta(\epsilon) = 8.8 \sqrt{\epsilon}$. 
Thus, using Corollary \ref{crl:ns1} we have
$$ \sum_{S\subseteq[n], |S| \geq \frac{1}{\beta^{-1}(\frac{\epsilon}{2.32})}}{\hat{f}^2(S)} \leq \epsilon$$
Noting the definition of the $\beta(.)$ function, we have $\beta^{-1}(x) = (\frac{x}{8.8})^2$, and in particular, $\beta^{-1}(\frac{\epsilon}{2.32}) = (\frac{\epsilon}{20.4166})^2 \geq  (\frac{\epsilon}{21})^2$. Therefore, $\frac{1}{\beta^{-1}(\frac{\epsilon}{2.32})} \leq  \frac{441}{\epsilon^2}$. Hence, we can conclude that
$$ \sum_{S\subseteq[n], |S| \geq \frac{441}{\epsilon^2}}{\hat{f}^2(S)} \leq \sum_{S\subseteq[n], |S| \geq \frac{1}{\beta^{-1}(\frac{\epsilon}{2.32})}}{\hat{f}^2(S)}\leq \epsilon$$

\end{proof}
%------------------------------------------------------------------------------------------------------------------------------------------------

Note that this corollary, along with our low-degree learning algorithm, gives us a $n^{O(\frac{1}{\epsilon^2})}$ sample-uniform learning algorithm for half-spaces. As a side note, we add that one can also learn half-space functions in polynomial time, via linear programming.
%------------------------------------------------------------------------------------------------------------------------------------------------
\subsection{Application: Learning Boolean Functions of $k$ Half-Space Functions}
\begin{theorem} \label{thm:hs}Consider half-space functions $h_1$, $h_2$, $\dots$, $h_k$, and boolean functions $f,g: \{\pm 1\}^{k} \rightarrow \{\pm 1\}$ such that $f(x) = g(h_1(x), h_2(x), \dots, h_k(x))$. Then, we have $\textit{ns}_{\epsilon}(f) \leq 8.8 k \sqrt{\epsilon}$.
\end{theorem}
%------------------------------------------------------------------------------------------------------------------------------------------------
\begin{proof}
The main idea of the proof is simple Union bound. We have 
\begin{eqnarray} \textit{ns}_{\epsilon}(f) &&= \Pr \big[f(x) \neq f(N_{\epsilon}(x))\big] \nonumber \\
&&= \Pr \big[g(h_1(x), h_2(x), \dots, h_k(x)) \neq g(h_1(N_{\epsilon}(x)), h_2(N_{\epsilon}(x)), \dots, h_k(N_{\epsilon}(x)))\big] \nonumber\\
&& \leq \Pr \big[ h_1(x) \neq h_1(N_{\epsilon}(x)) \; \textit{ or } \; h_2(x) \neq h_2(N_{\epsilon}(x)) \; \textit{ or } \; \dots \; \textit{ or } \; h_k(x) \neq h_k(N_{\epsilon}(x)) \big] \nonumber \\
&& \leq \Pr\big[ h_1(x) \neq h_1(N_{\epsilon}(x))\big] + \Pr\big[ h_2(x) \neq h_2(N_{\epsilon}(x))\big] + \dots + \Pr\big[ h_k(x) \neq h_k(N_{\epsilon}(x))\big] \nonumber \\
&& \leq 8.8 k \sqrt{\epsilon}
\end{eqnarray}
where the least inequality follows from Theorem \ref{thm:Peres}.
\end{proof}
%------------------------------------------------------------------------------------------------------------------------------------------------
\begin{corollary}Consider half-space functions $h_1$, $h_2$, $\dots$, $h_k$, and boolean functions $f,g: \{\pm 1\}^{k} \rightarrow \{\pm 1\}$ such that $f(x) = g(h_1(x), h_2(x), \dots, h_k(x))$. Then, we have $$ \sum_{S\subseteq[n], |S| \geq \Theta(\frac{k^2}{\epsilon^2})}{\hat{f}^2(S)} \leq \epsilon$$
\end{corollary}
%------------------------------------------------------------------------------------------------------------------------------------------------
\begin{proof} Follows immediately from Corollary \ref{crl:ns1} and Theorem \ref{thm:hs}.
\end{proof}
%------------------------------------------------------------------------------------------------------------------------------------------------

Again, this corollary, along with our low-degree learning algorithm, gives us a $n^{O(\frac{k^2}{\epsilon^2})}$ sample-uniform learning algorithm for any boolean function of $k$ half-space functions. Side note is that, in contrast to learning half-space functions, it is not known how to learn arbitrary functions of k half-space functions in polynomial time via linear programming.
\subsection{Learning Parity Functions}
In this model, we assume that there is a black box which contains an unknown parity function $f: \{0, 1\}^{n} \rightarrow \{0, 1\}$ and the goal is to learn this function. Speaking differently, the black box calculates some function $f = \chi_{S}$ and we want to figure out what $S$ is. For this, we have access to samples of this function, i.e., $(x_1, f(x_1))$, $(x_2, f(x_2))$, $\dots$, $(x_m, f(x_m))$.  

\paragraph{The case without noise:}If there is no noise, i.e., we have access to exact evaluations, then this problem can be solved simply by solving a system of linear equations, where we have $n$ unknowns such that the $i^{th}$ unknown is zero if $i\notin S$ and one otherwise.


\paragraph{The case with noise:}In the sequel, we  focus on the problem of learning parity functions in the presence of noise. This means that for each sample-point $x_i$, we have $\Pr\big[ f(x_i) = \chi_{S}(x_i)\big] \geq 0.5 + \delta$ where $\delta$ is some possibly small positive probability. In this scenario, the goal might be either to find $\chi_{S}(.)$ that is closest to $f(.)$ or find all $\chi_{S}(.)$  that are close enough to $f(.)$. The former case is just finding the largest Fourier coefficient while the latter is finding all large enough Fourier coefficients.

\medskip
There are different variants to this problem. When sample-points $x_1$, $x_2$, $\dots$, $x_m$ are chosen adverserially, the problem is equivalent to \emph{``maximum likelihood decoding of linear codes''} which is known to be NP-hard. When sample-points $x_1$, $x_2$, $\dots$, $x_m$ are chosen randomly at uniform, the problem is equivalent to \emph{``hardness of decoding linear codes''}, or also often called \emph{``hardness of parity with noise''}. A possibly slightly easier version asks what happens if the noise just flips every bit with some probability. This version is often called \emph{``hardness of decoding random linear codes''} which is usually used as an assumption in cryptography and coding. A theorem by Blum, Kalai, and Wasserman gives a slightly sub-exponential ($2^{\frac{n}{\log n}}$) algorithm for this variant. Next time, we will study the question that ``what happens if one can make queries into desired sample-points?''.

\end{document}




