%\input{mime.sty}
\documentclass[12pt,a4paper]{article}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{multirow}
\usepackage{listings}
\usepackage{url}
%\usepackage{subcaption}
\usepackage{subfig}
\usepackage{calc}
\usepackage{bm}
\usepackage{hyperref}
\date{} 
% Shortcuts
% ---------
\newcommand{\mw}{MetalWalls}

% Math formatting command
% -----------------------

% Vectors are in bold
\renewcommand{\vec}[1]{\bm{#1}}

% Define erf and erfc as math functions
\DeclareMathOperator{\erf}{erf}
\DeclareMathOperator{\erfc}{erfc}

% Define ceiling/floor function notation 
% if called as \ceil*{x} it will add \left and \right; you can also call it as \ceil[\big]{x} \ceil[\Big]{x} \ceil[\bigg]{x} \ceil[\Bigg]{x} to state explicitly the size of the delimiters.
\usepackage{mathtools}
\DeclarePairedDelimiter{\floor}{\lfloor}{\rfloor}
\DeclarePairedDelimiter{\ceil}{\lceil}{\rceil}

\begin{document}
\author{Abel Marin-Lafleche}
\title{Parallelization of Metalwalls}
\maketitle
\tableofcontents
%
%\chapter{Molecular Interactions}
%\label{ch:molecular-interactions}
%\include{molecular-interactions}
%
%\chapter{The MD algorithm}
%\label{ch:the-md-algorithm}
%\include{the-md-algorithm}
%
%\chapter{Constant Potential Simulation}
%\label{ch:constant-potential-simulation}
%\include{constant-potential-simulation}
%
%\chapter{Implementation}
%\label{ch:implementation}
%
% \chapter{Optimisation and HPC activities}
% \label{ch:performance}
% \include{perf}
%

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:
%\section{Parallelisation}
\label{sec:parallelisation}

\section{Long Range potential kernel}
The long range potential equation is derived from the 
energy equation as $V_{i}^{\mathrm{lr}} = \frac{\partial
  U_{c}^{\mathrm{lr,*}}}{\partial q_i}$. Using the formula
%given in Equation (\ref{eq:Uc-pp-lr-2dpbc}) this yields: 
given in the Ewald summation description, this yields: 

\begin{equation}
  V_{i}^{\mathrm{lr}} = \frac{2}{ab} \sum_{j} Q_j \sum_{\substack{(l,m)\in\mathbf{Z^2} \\ (l,m) \neq (0,0)}} \int_{-\infty}^{\infty} du\, 
  \frac{\cos \left(l\frac{2\pi}{a}x_{ij} + m\frac{2\pi}{b}y_{ij} + uz_{ij}\right)}{\left(l\frac{2\pi}{a}\right)^2 + \left(m\frac{2\pi}{b}\right)^2 + u^2}e^{-\frac{\left(l\frac{2\pi}{a}\right)^2 + \left(m\frac{2\pi}{b}\right)^2 + u^2}{4\alpha^2}} 
\end{equation}

The integral over $du$ is discretized using the midpoint rule with a
step size of $du = \frac{2\pi}{c}$. The term
$\int_{-\infty}^{\infty} du\,f(u)$ is therefore replaced by
$\frac{2\pi}{c} \sum_{n \in \mathbf{Z}}
f\left(n\frac{2\pi}{c}\right)$.

We call a \emph{k-point} the triplet
\begin{equation}
	k_{lmn} = (l\frac{2\pi}{a}, m\frac{2\pi}{a}, n\frac{2\pi}{c}) \text{ where }
(l,m,n) \in \mathbf{Z^3}. 
\end{equation}
	For large values of the k-point norm
$$|k_{lmn}| = \sqrt{\left(l\frac{2\pi}{a}\right)^2 +
  \left(m\frac{2\pi}{b}\right)^2 + \left(n\frac{2\pi}{c}\right)^2},$$
the term $\exp(\frac{-|k_{lmn}|^2}{4\alpha^2})$ is negligible. A
cut-off value $k_{\mathrm{max}}$ is used and only terms for which
$|k_{lmn}| < k_{\mathrm{max}}$ are taken into account. Thus the
infinite sums are truncated such that
$$|l| < \ceil*{k_{\mathrm{max}}\frac{a}{2\pi}},
|m| < \ceil*{k_{\mathrm{max}}\frac{b}{2\pi}} \text{ and }
|n| < \ceil*{k_{\mathrm{max}}\frac{c}{2\pi}}.$$

The term inside the sum is symmetric with respect to the $(l,m,n)$
triplets. Half of the computation effort is avoided by computing only
terms for which
\begin{align*}
(l=0,\allowbreak m \in [1,m_\mathrm{max}],\allowbreak n \in
[-n_\mathrm{max}, n_\mathrm{max}]) \text{ and} \\ 
(l \in [1,l_\mathrm{max}],\allowbreak m \in
[-m_\mathrm{max},m_\mathrm{max}],\allowbreak n \in [-n_\mathrm{max},
n_\mathrm{max}]).
\end{align*}

Trigonometric rules are used to reduce the complexity of computing all
values of $V_{i}^{\mathrm{lr}}$ from $\mathcal{O}(N^2)$ to simply
$\mathcal{O}(N)$. Using the equality
$\cos(a-b) = \cos(a)\cos(b) + \sin(a)\sin(b)$, it appears that for
each $(l,m,n)$ triplet the same factors are used for all $i$:
\begin{align*}
C_{l,m,n} = \sum_{j}Q_j \cos \left(l\frac{2\pi}{a}x_{j} +
  m\frac{2\pi}{b}y_{j} + n\frac{2\pi}{c}z_{j}\right) \\
S_{l,m,n} = \sum_{j}Q_j \sin \left(l\frac{2\pi}{a}x_{j} +
  m\frac{2\pi}{b}y_{j} + n\frac{2\pi}{c}z_{j}\right)
\end{align*}

In order to avoid the call to the expensive intrinsic function $\cos$
and $\sin$, the required values are pre-computed and stored in six
arrays: 
\begin{align*}
C_{lj} = \cos(l\frac{2\pi}{a}x_{j}),
S_{lj} = \sin(l\frac{2\pi}{a}x_{j}),\\
C_{mj} = \cos( m\frac{2\pi}{b}y_{j}),
S_{mj} = \sin(m\frac{2\pi}{b}y_{j}),\\
C_{nj} = \cos(n\frac{2\pi}{c}z_{j}),
S_{nj} = \sin(n\frac{2\pi}{c}z_{j}). 
\end{align*}
Then, using trigonometric rules, the values of
\begin{align*}
C_{lmnj}= C_{lj}C_{mj}C_{nj} - S_{lj}C_{mj}C_{nj} -
C_{lj}S_{mj}C_{nj} - C_{lj}C_{mj}S_{nj},\\ 
S_{lmnj}= S_{lj}C_{mj}C_{nj} + C_{lj}S_{mj}C_{nj} +
C_{lj}C_{mj}S_{nj} - S_{lj}S_{mj}S_{nj} 
\end{align*}
are easily recovered. This
trick requires a storage of
$2N(l_\mathrm{max} + m_\mathrm{max} + n_\mathrm{max} + 3)$ double
precision words when we store only values for positive $l$, $m$ and
$n$. One can note that storing all values of $C_{lmnj}$ and $S_{lmnj}$
would require
$2N(2l_\mathrm{max}m_\mathrm{max}+ l_\mathrm{max}+
m_\mathrm{max})(2n_\mathrm{max}+1)$ double precision words.

The long range potential computed in the Metalwalls implementation
corresponds to

\begin{equation}
\tilde{V}_{i}^{\mathrm{lr}} = \frac{8\pi}{abc} \sum_{|k_{lmn}|<k_{\mathrm{max}}} (C_{lmni}C_{lmn} + S_{lmni}S_{lmn})   \frac{e^{-\frac{|k_{lmn}|^2}{4\alpha^2}} }{|k_{lmn}|^2}
\end{equation}

\begin{figure}
  \caption{\label{lr-serial-listing} Long range potential kernel fortran implementation}
  \includegraphics[width=0.9\textwidth]{figures/kernel_coulomb_lr.pdf}
\end{figure}

The listing of the source code corresponding to this implementation is
shown in Figure \ref{lr-serial-listing}. The kernel consists in two
loop levels. The outer loop runs on the k-points and two inner loops
run on the number of electrode atoms. The first inner loop
corresponds to a reduction on the variables \texttt{Sk\_cos} and
\texttt{Sk\_sin}. The second inner loop uses these values to add the
contribution of the current k-point to each electrode atom
potential. The two inner loops are executed only if the k-point
satisfies the cut-off criterion. The subroutine
\texttt{compute\_kmode\_index} computes the $(l,m,n)$ triplet
corresponding to a given k-point index.

As an optimization, blocking has been introduced in order to reuse
data already loaded into the cache. Indeed, when the k-point index,
\texttt{imode}, increases, the $n$ index increases the fastest while
the $l$ index increases the slowest. Therefore the data loaded from
\texttt{cos\_kx\_elec(j,l)}, \texttt{sin\_kx\_elec(j,l)},
\texttt{cos\_ky\_elec(j,m)}, \texttt{sin\_ky\_elec(j,m)} can be
reused. Figure \ref{lr-blocking-listing} shows the listing of the
source code corresponding to this optimization. In order, to save some
space, parts of the code which are identical to the one presented in
Figure \ref{lr-serial-listing} has been replaced by ellipses. As can
be seen, the kernel is now decomposed into two loop nests of
three-level. The outer level corresponds to the blocking, the second
level is on the k-point and the innermost level is on the electrode
atoms. The size of the innermost loop is controled by a constant named
\texttt{block\_vector\_size}. We also have to introduce the two 1D
arrays \texttt{Sk\_cos(:)} and \texttt{Sk\_sin(:)} of size
\texttt{num\_kpoints} to be able to use the values computed in the
first loop nest in the second one.

\begin{figure}
  \caption{\label{lr-blocking-listing} Long range potential kernel fortran implementation with blocking}
  \includegraphics[width=0.9\textwidth]{figures/kernel_coulomb_lr_blocking.pdf}
\end{figure}

The computation of the contribution from one k-point is completely
independent from the other k-points. The MPI parallelisation strategy
is straightforward. Data is replicated on each process and work is
distributed. Each process is assigned a range of k-points to work on
and a call to \texttt{MPI\_Allreduce} is made at the end of the kernel
to sum up all the k-points contribution into the vector
\texttt{V(:)}. During the setup phase, k-points which satisfy the
cut-off criteria are assigned a weight of 1 and k-points which don't
satisfy the cut-off criteria are assigned a weight of 0. The k-points
are distributed consecutively to each processes such that all
processes work on an equal amount of weighted k-points. Figure
\ref{lr-blocking-mpi-listing} shows the skeleton of the implementation
for the MPI implementation of this kernel. The loop on the k-point
index now runs only the local range, from \texttt{imode\_start} to
\texttt{imode\_end}. The local and global representation of the
potential values \texttt{V(:)} is explicitely shown, however one can
avoid the allocation of the extra storage by using the
\texttt{MPI\_IN\_PLACE} tag in the call to
\texttt{MPI\_Allreduce}. The \texttt{Sk\_cos(:)} and
\texttt{Sk\_sin(:)} arrays are local to each process and thus they
only need to be allocated for the range
\texttt{(imode\_start:imode\_end)}.

\begin{figure}
  \caption{\label{lr-blocking-mpi-listing} Long range potential kernel fortran implementation with MPI parallelization}
  \includegraphics[width=0.9\textwidth]{figures/kernel_coulomb_lr_blocking_mpi.pdf}
\end{figure}


\newpage 

\section{$k=0$ potential kernel}
The $k=0$ potential equation is derived from the 
energy equation as $V_{i}^{k=0} = \frac{\partial
  U_{c}^{\mathrm{lr,0}}}{\partial qi}$. Using the formula
%given in Equation (\ref{eq:Uc-pp-k0-2dpbc}) this yields: 
given in the Ewald summation document, this yields: 

\begin{equation}
  V_{i}^{k=0} = - \frac{2\sqrt{\pi}}{ab} \sum_{j} Q_j  \left(\frac{e^{-z_{ij}^2 \alpha^2}}{\alpha} + \sqrt{\pi} |z_{ij}| \erf(\alpha |z_{ij}|) \right)
\end{equation}

The $k=0$ potential on an electrode atom is the sum of the
contribution from all other atoms in the system. One can note that for
a pair of atoms $\frac{V_{ij}^{k=0}}{Q_j} = \frac{V_{ji}^{k=0}}{Q_i}$
and therefore the computation of the potential on each electrode atom
can be implemented as a triangular loop as shown in Figure
\ref{keq0-listing}.

\begin{figure}
\caption{\label{keq0-listing} $k=0$ potential kernel fortran serial implementation}
\includegraphics[width=0.9\textwidth]{figures/kernel_coulomb_keq0.pdf}
\end{figure}

In order to improve the data locality and the vectorization potential,
the triangular loop on atom pairs has been reformulated into a loop of
block of atoms pairs. The block size parameter is a constant parameter
chosen to be a multiple of the vector length. Figure
\ref{pair-triangle} shows the order used to compute pair interaction
in the triangular loop, while Figure \ref{pair-block} shows the
ordering used in the blocked version. All blocks are treated fully,
even blocks on the diagonal for which we don't use the symmetry of the
interaction. For small values of the block size parameter, the extra
work is compensated by an increase in vectorization potential. The
corresponding implementation is shown in Figure
\ref{keq0-block-listing}.

\begin{figure}
  \centering
  \caption{\label{pair-triangle} Pair ordering for triangular loop nest (\texttt{num\_atoms}=12)}
  \includegraphics[width=8cm]{figures/pair_triangle_interaction.pdf}
\end{figure}

\begin{figure}
\centering
\caption{\label{pair-block} Pair ordering for block loop nest}
\includegraphics[width=8cm]{figures/pair_block_interaction.pdf}
\end{figure}

\begin{figure}
\caption{\label{keq0-block-listing} $k=0$ potential kernel fortran blocked implementation}
\includegraphics[width=0.9\textwidth]{figures/kernel_coulomb_keq0_block.pdf}
\end{figure}

From this point, parallelisation in MPI is straightforward. Using a
data replication strategy, the work is distributed by blocks. Each
process computes a contribution from a subset of the blocks and a call
to \texttt{MPI\_Allreduce} is used to add the contribution from all
blocks to the potential. The implementation is shown in Figure
\ref{keq0-block-mpi-listing}.

\begin{figure}
  \caption{\label{keq0-block-mpi-listing} $k=0$ potential kernel fortran parallel implementation}
  \includegraphics[width=0.9\textwidth]{figures/kernel_coulomb_keq0_block_mpi.pdf}
\end{figure}

\newpage

\section{Short Range potential kernel}
The short range potential equation is derived from the 
energy equation as $V_{i}^{\mathrm{sr}} = \frac{\partial
  U_{c}^{\mathrm{sr}}}{\partial qi}$. Using the formula
%given in Equation (\ref{eq:Uc-pp-sr-2dpbc}) this yields: 
given in the Ewald summation document this yields: 

\begin{equation}
  \label{eq:short-potential}
  V_{i}^{\mathrm{sr}} = \sum_{j} Q_j \sideset{}{'}\sum_{\vec{n}} |\vec{r_{ij}} + \vec{n}|^{-1} \left( \erfc(\alpha|\vec{r_{ij}} + \vec{n}|) - \erfc (\eta_{ij}|\vec{r_{ij}} + \vec{n}|)\right)
\end{equation}

In Equation (\ref{eq:short-potential}), the sums run on all particles
and all of their images. The $'$ indicates that the self-interaction
term should be ommited. However, we cannot compute infinite sums on a
computer. A cut-off distance distance $r_{cut}$ is imposed and the
minimum image distance convention is used.

The structure of the short range potential kernel is very close to the
$k=0$ potential. The interaction are symmetric and thus involve a
triangular loop. The main difference, apart from the actual value, is
the presence of a cut-off radius and the absence of self-interaction
which may hinder vectorization. However the same blocking and
parallelisation strategy is used as in the other kernel. The Figure
\ref{sr-block-mpi-listing} shows the fortran implementation of the
kernel.

Knowing that electrode atoms have a constant position during the whole
simulation. It is possible to distribute only blocks for which we
know, from a setup phase computation, where there will be some
interactions between particles in those blocks.

\begin{figure}
\caption{\label{sr-block-mpi-listing} Short-range potential kernel fortran parallel implementation}
\includegraphics[width=0.9\textwidth]{figures/kernel_coulomb_sr_block_mpi.pdf}
\end{figure}

\newpage

\section{Self potential kernel}
The short range potential equation is derived from the 
energy equation as $V_{i}^{\mathrm{sr}} = \frac{\partial
  U_{c}^{\mathrm{self}}}{\partial qi}$. Using the formula
%given in Equation (\ref{eq:Uc-pp-self-2dpbc}) this yields: 
given in the Ewald summation document this yields: 

\begin{equation}
  V_{i}^{\mathrm{self}} = \frac{2}{\sqrt{\pi}} Q_i \left( \frac{\eta_{i}}{\sqrt{2}} - \alpha \right)
\end{equation}

The self potential kernel is straightforward to implement. It is
simply a vector scaling. It will not be discussed further in this
document.

\bibliography{metalwalls}
\bibliographystyle{plain}
\end{document}
