
%% Kluwer Small Journal Article Template File
%% Current version: January 24, 1994

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Prepared by Amy Hendrickson, TeXnology Inc.                          %%
%% (617) 738-8029                                                       %%
%%                                                                      %%
%% Inquiries to Suzanne M. Rumsey, net address: prod@world.std.com      %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


% Use one of these two commands: 
\documentstyle[theapa]{kbsjrnl} % For Computer Modern Fonts, or,
% \documentstyle[kbsfonts]{kbsjrnl} % for PostScript fonts
%\renewcommand{\floatpagefraction}{0.9}

\begin{document}
\input{psfig}
\bibliographystyle{theapa}
\begin{article}


%%%%% To be entered at Kluwers: =====>>
%\journame{Machine Learning}
%\volnumber{}
%\issuenumber{}
%\issuemonth{}
%\volyear{}

%% Do not delete either of the following two commands.
%% Please supply facing curly brackets for the part you
%% are not using for this article.
%\received{November 1, 1994}\revised{August 1, 1995}
%
%\authorrunninghead{R. MACLIN AND J. W. SHAVLIK}
%\titlerunninghead{CREATING ADVICE-TAKING REINFORCEMENT LEARNERS}
%
%\setcounter{page}{275} %% Optional, uncomment to set page number.
%%%%  <<==== End of commands to be entered at Kluwers 


%%  Authors, start here ====>>

%\title{Creating Advice-Taking Reinforcement Learners}
%
%% Author name 
%\authors{Richard Maclin and Jude W. Shavlik}

%% Email address. Do not delete this command even if you do not have
%% an email address; just leave the command followed by facing curly brackets.
%\email{maclin@cs.wisc.edu and shavlik@cs.wisc.edu}

%% Affiliation address
%\affil{Computer Sciences Dept., University of Wisconsin, 1210 W. Dayton St., Madison, WI  53706}

%% Article editor
%\editor{Leslie Pack Kaelbling}

%% Abstract
%\abstract

%% Keywords
%\keywords{Reinforcement learning, advice-giving, neural networks, Q-learning, learning from instruction, theory refinement, knowledge-based neural networks, adaptive agents}

  \begin{figure}
    \centerline{\psfig{figure=figure1.ps,width=\textwidth}}
    \vspace{1.0in}
    \caption{In basic reinforcement learning the learner receives a description of the current environment (the state), selects an action to choose, and receives a reinforcement as a consequence of selecting that action.  We augment this with a process that allows an observer to watch the learner and suggest advice based on the learner's behavior.}
    \label{advice:qlearner}
  \end{figure}

  \newpage

  \begin{specialtable}
    \caption{Samples of advice in our advice language (left column).}
    \vspace{1.0in}
    \begin{tabular}{@{}p{1.5in}@{~~~}p{1.3in}@{~~~}p{1.3in}@{}}
      \multicolumn{1}{c}{Advice} & \multicolumn{1}{c}{English Version} & \multicolumn{1}{c}{Pictorial Version} \\ \hline
      {
	\vspace{-0.79in}
        \begin{tabbing}
          {\sc if} \= An Obstacle \= {\sc is} (Near $\wedge$ North) \kill
          {\sc if} \> An Enemy    \> {\sc is} (Near $\wedge$ West) $\wedge$ \\
             \> An Obstacle \> {\sc is} (Near $\wedge$ North) \\
          {\sc then} \\
                \> {\sc multiaction} \\
                \> {\sc if} \= \kill
                \> \> MoveEast \\
                \> \> MoveNorth \\
                \> {\sc end} \\
	  {\sc end}; \\
        \end{tabbing}
      }
      &
      {
	\vspace{-0.65in}
          If an enemy is near and west and an obstacle is near
          and north, hide behind the obstacle.
      }
      &
      \psfig{figure=table1-picture1.ps,width=1.3in}
      \\
      {
	\vspace{-1.22in}
        \begin{tabbing}
          {\sc until} \= {\sc if} \= \kill
          {\sc when}  \> Surrounded $\wedge$ \\
		\> OKtoPushEast $\wedge$ \\
		\> An Enemy {\sc is} Near \\
          {\sc repeat} \\
		\> {\sc multiaction} \\
                \>    \> PushEast \\
                \>    \> MoveEast \\
		\> {\sc end} \\
          {\sc until} \> $\neg$ OKtoPushEast $\vee$ \\
		\> $\neg$ Surrounded \\
	  {\sc end}; \\
        \end{tabbing}
      }
      &
      {
	\vspace{-1.04in}
          When the agent is surrounded, pushing east is possible, and an enemy is
          near, then keep pushing (moving the obstacle out of the way) and moving
          east until there is nothing more to push or the agent is no longer
          surrounded.
      }
      &
      \psfig{figure=table1-picture2.ps,width=1.3in}
      \\
      {
	\vspace{-0.20in}
        \begin{tabbing}
          {\sc if} \= An Enemy {\sc is} (Near $\wedge$ East) \\
          {\sc then} \\
             \> {\sc do\_not} MoveEast \\
	  {\sc end}; \\
        \end{tabbing}
      }
      &
      {
        \vspace{-0.07in}
	  Do not move toward a nearby enemy.
      }
      &
      \vspace{-0.10in}
      \psfig{figure=table1-picture3.ps,width=1.3in}
    \end{tabular}
    \label{table:sample-rules}
%    \vspace{-0.15in}
  \end{specialtable}

  \newpage

  \begin{figure}
    \centerline{\psfig{figure=figure2.ps,width=\textwidth}}
    \vspace{1.0in}
    \caption{Adding advice to the RL agent's neural network by creating new hidden units that represent the advice.  The thick links on the right capture the semantics of the advice.  The added thin links initially have near-zero weight; during subsequent backpropagation training the magnitude of their weights can change, thereby refining the original advice.  Details and an example appear in Section~3.}
    \label{figure:add-advice}
  \end{figure}

  \newpage

  \begin{figure}
    \centerline{\psfig{figure=figure3.ps,width=\textwidth}}
    \vspace{1.0in}
    \caption{Interaction of the observer, agent, and our advice-taking system.  The process is a cycle: the observer watches the agent's behavior to determine what advice to give, the advice-taking system processes the advice and inserts it into the agent, which changes the agent's behavior, thus possibly causing the observer to provide more advice.  The agent operates as a normal Q-learning agent when not presented with advice.}
    \label{figure:system-diagram}
  \end{figure}

  \newpage

  \begin{figure}
    \centerline{\psfig{figure=figure4.ps,width=\textwidth}}
    \vspace{1.0in}
    \caption{Sample of the {\sc kbann} algorithm:  (i) a propositional rule set in Prolog notation; (ii) the rules viewed as an {\sc and-or} dependency graph; (iii) each proposition is represented as a unit (extra units are also added to represent disjunctive definitions, e.g., $b$), and their weights and biases are set so that they implement {\sc and} or {\sc or} gates, e.g, the weights $b \rightarrow a$ and $c \rightarrow a$ are set to 4 and $a$'s bias (threshold) to 6 (the bias of an {\sc or} node is 2); (iv) low-weighted links are added between layers as a basis for future learning (e.g., an antecedent can be added to a rule by increasing one of these weights).}
    \label{picture:kbann}
  \end{figure}

  \newpage

  \begin{figure}
    \begin{tabular}{@{}p{1.9in}@{~~}p{3.0in}@{}}
      {\footnotesize
	\vspace{-1.5in}
        \begin{tabbing}
          \hspace{0.1in} \= {\sc if} \= An Obstacle \= {\sc is} (Near $\wedge$ North) \kill
          \> {\sc if} \> An Enemy    \> {\sc is} (Near $\wedge$ West) $\wedge$ \\
          \>   \> An Obstacle \> {\sc is} (Near $\wedge$ North) \\
          \> {\sc then} \\
          \>   \> {\sc multiaction} \\
          \>   \> {\sc if} \= \kill
          \>   \> \> MoveEast \\
          \>   \> \> MoveNorth \\
          \>   \> {\sc end} \\
	  \> {\sc end}; \\
        \end{tabbing}
      }
      &
      \psfig{figure=figure5-picture.ps,width=3.0in}
    \end{tabular}
    \vspace{1.0in}
    \caption{On the left is the first piece of advice from Table~\protect\ref{table:sample-rules}.  On the right is {\sc ratle}'s translation of this piece of advice.  The shaded ellipse represents the original hidden units.  Arcs show units and weights that are set to implement a conjunction.  {\sc Ratle} adds zero-weighted links (not shown here -- see Figure~\protect\ref{picture:kbann}d) to other parts of the current network; these links support subsequent refinement.}
    \label{figure:advice-hide}
  \end{figure} 

  \newpage

  \begin{figure}
    \begin{tabular}{@{}p{1.9in}@{~~}p{3.0in}@{}}
      {\footnotesize
	\vspace{-1.9in}
        \begin{tabbing}
          \hspace{0.1in} \= {\sc until} \= {\sc if} \= \kill
          \> {\sc when}  \> Surrounded $\wedge$ \\
	  \>	\> OKtoPushEast $\wedge$ \\
	  \>	\> An Enemy {\sc is} Near \\
          \> {\sc repeat} \\
	  \>	\> {\sc multiaction} \\
          \>    \>    \> PushEast \\
          \>    \>    \> MoveEast \\
	  \>	\> {\sc end} \\
          \> {\sc until} \> $\neg$ OKtoPushEast $\vee$ \\
	  \>	\> $\neg$ Surrounded \\
	  \> {\sc end}; \\
        \end{tabbing}
      }
      &
      \psfig{figure=figure6-picture.ps,width=3.00in}
    \end{tabular}
    \vspace{1.0in}
    \caption{On the left is the second piece of advice from Table~\protect\ref{table:sample-rules}. On the right is {\sc Ratle}'s translation of it.  Dotted lines indicate negative weights.  These new units are added to the existing network (not shown).}
    \label{figure:advice-surrounded}
    \vspace{1.0in}
  \end{figure}

  \newpage

  \begin{figure}
    \centerline{\psfig{figure=figure7.ps,width=\textwidth}}
    \vspace{1.0in}
    \caption{Incorporating the definition of a term that already exists.}
    \vspace{1.0in}
    \label{figure:add-to-existing} 
  \end{figure}

  \newpage

  \begin{figure}
    \centerline{\psfig{figure=figure8.ps,width=\textwidth}}
    \vspace{1.0in}
    \caption{Our sample test environment: (a) sample configuration; (b) sample division of the environment into sectors; (c) distances to the nearest occluding object along a fixed set of arcs (measured from the agent); (d) a neural network that computes the utility of actions.}
    \label{fig:sample-pengo}
  \end{figure}

  \newpage

  \begin{figure}
    \centerline{\psfig{figure=figure9.ps,width=\textwidth,bbllx=74pt,bblly=90pt,bburx=517pt,bbury=403pt}}
    \vspace{1.0in}
    \caption{Average total reinforcement for our four sample pieces of advice as a function of amount of training and point of insertion of the advice.}
    \label{graph:both-results}
  \end{figure}

  \newpage

  \begin{figure}
    \centerline{\psfig{figure=figure10.ps,width=\textwidth}}
    \vspace{1.0in}
    \caption{A sample problem where good advice can fail to enhance performance.  Assume the goal of the agent is to go from Work to Home, and that the agent will receive a large reward for taking Nakoma Road, and a slightly smaller reward for taking University Avenue followed by Midvale Boulevard.  If the agent receives advice that University followed by Midvale is a good plan, the agent, when confronted with the problem of going from Work to Home will likely follow this plan (even during training, since actions are selected proportional to their predicted utility).  Thus it may take a long time for the learner to try Nakoma Road often enough to learn that it is a better route.  A learner without advice might try both routes equally often and quickly learn the correct utility value for each route.}
    \label{figure:advice-fails}
  \end{figure}

%% End matter:

%\acknowledgements

% \begin{references}
% \bibitem{xxx}
% \end{references}

%\bibliography{maclin-shavlik-mlj}

%% Do not delete this! ===>>>
\end{article}
\end{document}


% Samples of commands you may use:

\section{ }
\subsection{ }
\subsubsection{ }

%% If you want to make an equation that is not indented, use
%% \begin{wideequation} ... \end{wideequation}

% \begin{wideequation}
% \begin{equation} or \[
% <math>
% \end{equation} or \]
% \end{wideequation}

% In this wide equation the `array' command is used to split 
% the math into two lines, moving the top half to the left
% and the bottom to the right.
% \begin{wideequation}
% \begin{array}{lr}
% \sum_k P(k) \sum_i \sum_y f_i(y|k)^2\\
% &\sum_k P(k) \sum_i \sum_y f_i(y|k)^2
% \sum_k P(k) \sum_i \sum_y f_i(y|k)^2
% \end{array}
% \end{wideequation}


% To indent text, use the following commands:
% \begin{itemize}
% \item[] 
% text...
% \end{itemize}


%% Algorithm for exhibiting code. Indent lines with one or more `\ '.
% \begin{algorithm}
% \ start line here
% \ \ indent line here
% \end{algorithm}


% Sample figure
% \begin{figure}[h]
% \vspace*{.5in}
% \caption{This is a figure caption.
% This is a figure caption.
% This is a figure caption.}
% \end{figure}

% Sample table, this kind of table preamble will spread 
% table out to the width of the page:
% \begin{table}[h]
% \caption{This is an example table caption. As you can
% see, it will be as wide as the table that it captions.}
% \begin{tabular*}{\textwidth}{@{\extracolsep{\fill}}lcr}
% \hline
% $\alpha\beta\Gamma\Delta$ One&Two&Three\cr
% \hline
% one&two&three\cr
% one&two&three\cr
% \hline
% \end{tabular*}
% \end{table}

%% Make examples:
% \begin{example}
% text...
% \end{example}

%% Make theorems:
% \begin{proclaim}{Theorem <number>}
% text...
% \end{proclaim}

%% Make proof:
% \begin{proof}
% text...
% \end{proof}

% If proof ends with math, please use \inmathqed at the end of the
% equation:
% \begin{proof}
% ....
% \[
% \alpha\beta\Gamma\Delta\inmathqed
% \]
% \end{proof}

%% Proof with a title. Enter name of proof in square brackets:
% \begin{proof}[Proof of Theorem A.1]
% ...
% \end{proof}

%% Assumption or similar kind of environment
% \begin{demo}{Assumption <number>}
% text...
% \end{demo}

%% End Matter:

%%%%%%%
%% Acknowledgements here

% \acknowledgements
% text...

% Appendices:
% \appendix

%%%%%%%
%% Endnotes

%\notes

%%%%%%%
%% Make references as standard Latex.
%% for example:

%% Trying `cite', \cite{jacobs}, \cite{francis}.

% \begin{references}
% \bibitem{jacobs}Jacobs, E., ``Design Method Optimizes Scanning
% Phased Array,'' Microwaves, April 1982, pp.\ 69--70.

% \bibitem{francis} Francis, M., ``Out-of-band response of array antennas,''
% Antenna Meas.  Tech. Proc., September 28--October 2, 1987, Seattle, p.~14.
% \end{references}

% Or, to make alphabetical references, with no number preceding entries:

% Maude Francis, (Francis, 87) showed important new results
% with array antennas.

% \begin{alphareferences}
% Francis, M., ``Out-of-band response of array 
% antennas,'' Antenna Meas.  Tech. Proc., September 28--October 2,
% 1987, Seattle, p.~14.

% Jacobs, E., ``Design Method Optimizes Scanning
% Phased Array,'' Microwaves, April 1982, pp.\ 69--70.
% \end{alphareferences}

See kbsjrnl.ins for documentation on using Bibtex,
also on making an alphabetical reference section using Bibtex.
