%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% -*-latex-*-
%
%
% Synopsis:  XSEDE'12 Uintah GPU Scheduler/Ray Tracer Final Submission
% 
% Authors: Alan Humphrey, Qingyu Meng, Martin Berzins and Todd Harman
%
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%\documentclass[prepare]{acmconf}
\documentclass{sig-alternate}
\usepackage{cite,color,graphicx}
\usepackage{url}
\usepackage{times}
\usepackage{latexsym}
\usepackage{amsmath}
\usepackage{color}
\usepackage{wrapfig}
\usepackage{fancyhdr}
\usepackage{fancybox}
\usepackage{ifpdf}
\usepackage{array}
\usepackage{epstopdf}
\usepackage{algorithm}
\usepackage{algorithmic}
%
\graphicspath{{figures/}}
\newcommand{\Red}[1]{ {\color{red} #1} }

% correct bad hyphenation here
\hyphenation{processing}

% stop hyphenation of upper-case words and acronyms
\uchyph=0

% user defined to work with copyright boilerplate in sig-alternate.cls (May, 2012)
\copyrtyr={2012}
\acmcopyr={978-1-4503-0888-5/11/07}

%\ifpdf
%  \usepackage[pdftex]{graphicx}
%  \pdfcompresslevel=9
%  \DeclareGraphicsExtensions{{.png},{.pdf},{.jpg},{jpeg}}
%  \graphicspath{ {png/} {jpg/} {pdf/} }
%\else
%  \usepackage[dvips]{graphicx}
%  \DeclareGraphicsExtensions{.png.eps}
%  \graphicspath{ {fig/} {eps/} }
%\fi

%=================================================================
\begin{document}
%=================================================================

\conferenceinfo{XSEDE'12,}
{July 16-20, 2012, Chicago, Illinois, USA.}

\title{Radiation Modeling Using the Uintah Heterogeneous CPU/GPU Runtime System}

\numberofauthors{4}

\author{
% You can go ahead and credit any number of authors here,
% e.g. one 'row of three' or two rows (consisting of one row of three
% and a second row of one, two or three).
%
% The command \alignauthor (no curly braces needed) should
% precede each author name, affiliation/snail-mail address and
% e-mail address. Additionally, tag each line of
% affiliation/address with \affaddr, and tag the
% e-mail address with \email.
%
% 1st. author
\alignauthor Alan Humphrey\\
       \affaddr{Scientific Computing and Imaging Institute}\\
       \affaddr{University of Utah }\\
       \affaddr{Salt Lake City, UT 84112 USA}\\
       \email{ahumphrey@sci.utah.edu}
% 2nd. author
\alignauthor Qingyu Meng\\
       \affaddr{Scientific Computing and Imaging Institute}\\
       \affaddr{University of Utah }\\
       \affaddr{Salt Lake City, UT 84112 USA}\\
       \email{qymeng@sci.utah.edu}
% 3rd. author
\alignauthor Martin Berzins\\
       \affaddr{Scientific Computing and Imaging Institute}\\
       \affaddr{University of Utah }\\
       \affaddr{Salt Lake City, UT 84112 USA}\\
       \email{mb@sci.utah.edu}
\and
%\and  % use '\and' if you need 'another row' of author names
% 4th. author
\alignauthor Todd Harman\\
       \affaddr{Department of Mechanical Engineering}\\
       \affaddr{University of Utah }\\
       \affaddr{Salt Lake City, UT 84112 USA}\\
       \email{t.harman@utah.edu}
}
\date{July 2012}

\maketitle

%______________________________________________________________________
%

\begin{abstract}
The Uintah Computational Framework was developed to provide an environment for
solving fluid-structure interaction problems on structured adaptive grids on
large-scale, long-running, data-intensive problems. Uintah uses a combination
of fluid-flow solvers and particle-based methods for solids, together with a
novel asynchronous task-based approach with fully automated load balancing.
Uintah demonstrates excellent weak and strong scalability at full machine
capacity on XSEDE resources such as Ranger and Kraken, and through the use of
a hybrid memory approach based on a combination of MPI and Pthreads, Uintah now
runs on up to 262k cores on the DOE Jaguar system. In order to extend Uintah to
heterogeneous systems, with ever-increasing CPU core counts and additional on-node GPUs,
a new dynamic CPU-GPU task scheduler is designed and evaluated in this study.
This new scheduler enables Uintah to fully exploit these architectures with
support for asynchronous, out-of-order scheduling of both CPU and GPU computational
tasks. A new runtime system has also been implemented with an added multi-stage
queuing architecture for efficient scheduling of CPU and GPU tasks. This new runtime
system automatically handles the details of asynchronous memory
copies to and from the GPU and introduces a novel method of pre-fetching
and preparing GPU memory prior to GPU task execution. In this study this new design
is examined in the context of a developing, hierarchical GPU-based ray tracing radiation
transport model that provides Uintah with additional capabilities for heat
transfer and electromagnetic wave propagation. The capabilities
of this new scheduler design are tested by running at large scale on
the modern heterogeneous systems, Keeneland and TitanDev, with up to 360 and 960 GPUs respectively.
On these systems, we demonstrate significant speedups per GPU against a standard CPU core
for our radiation problem.
\end{abstract}

%______________________________________________________________________
%

% A category with the (minimum) three required fields
\category{D.1.3}{Software}{Concurrent Programing}
\category{G.1.8}{Mathematics of Computing}{Partial Differential Equations}
\category{G.4}{Mathematics of Computing}{Mathematical Software}
% A category including the fourth, optional field follows...
\category{J.2}{Computer Applications}{Physical Sciences and Engineering}

%______________________________________________________________________
%

\terms{Design, Performance, Algorithms}

%______________________________________________________________________
%

\begin{keywords}
Uintah, hybrid parallelism, scalability, parallel, adaptive, GPU,
heterogeneous systems, Keeneland, TitanDev
\end{keywords}

%______________________________________________________________________
%

\section{Introduction}
\label{subsec:summ-features}
\input{introduction.tex}

%______________________________________________________________________
%

\section{Overview of Uintah Software}
\label{sec:uintah-overview}
\input{uintah-overview.tex}

%______________________________________________________________________
%

\section{The ARCHES Combustion\\*Simulation Component}
\label{sec:arches-component}
\input{arches-component.tex}

%______________________________________________________________________
%

\section{Scheduler Architecture}
\label{sec:runtime-design}
\input{runtime-design.tex}

%______________________________________________________________________
%

\section{Computational Experiments}
\label{sec:computational-experiments}
\input{computational-experiments.tex}

%______________________________________________________________________
%

\section{Future Work}
\label{sec:future-work}
\input{future-work.tex}

%______________________________________________________________________
%

\section{Acknowledgments}
This work was supported by the National Science Foundation under subcontracts
No. OCI0721659, the NSF OCI PetaApps program, through award OCI 0905068 and by 
DOE INCITE award CMB015 for time on Jaguar and DOE NETL for funding under
NET DE-EE0004449. Uintah was written by the University of Utah's Center for the
Simulation of Accidental Fires and Explosions (C-SAFE) and funded by the Department
of Energy, subcontract No. B524196. This research used resources of the Keeneland
Computing Facility at the Georgia Institute of Technology, which is supported by
the National Science Foundation under Contract OCI-0910735. We would like to thank
Dr. Jeff Vetter and the Keeneland team for access to Keeneland resources and the
support provided. This research also used resources from the TitanDev program, the 960-node
partition of the Cray XK6 Jaguar system outfitted with Nvidia GPUs located at Oak
Ridge National Laboratory (ORNL). We would like to thank ORNL, Dr. Jack Wells and
the Resource Utilization Council at the Oak Ridge Leadership Computing Facility
for granting our project early access to TitanDev resources.
We would also like to thank all those previously involved with Uintah, Justin
Luitjens, Jeremy Thornock, Isaac Hunsaker and Joseph Peterson in particular.

%______________________________________________________________________
%

\bibliographystyle{plain}
\bibliography{uintah_gpu_xsede12}

%=================================================================
\end{document}
%=================================================================
