Commits

Eric Chen  committed 14ae422

add sparse autoencoder documentation.

  • Participants
  • Parent commits c78fc81

Comments (0)

Files changed (16)

File deepArch/_modules/CostFunc.html

     <script type="text/javascript" src="../_static/jquery.js"></script>
     <script type="text/javascript" src="../_static/underscore.js"></script>
     <script type="text/javascript" src="../_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="../index.html" />
     <link rel="up" title="Module code" href="index.html" /> 
   </head>
         <li class="right" style="margin-right: 10px">
           <a href="../genindex.html" title="General Index"
              accesskey="I">index</a></li>
-        <li class="right" >
-          <a href="../py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="../index.html">deepArch 0.1 documentation</a> &raquo;</li>
           <li><a href="index.html" accesskey="U">Module code</a> &raquo;</li> 
       </ul>
             
   <h1>Source code for CostFunc</h1><div class="highlight"><pre>
 <span class="c"># -*- coding: UTF-8 -*-</span>
-<span class="kn">from</span> <span class="nn">numpy</span> <span class="kn">import</span> <span class="nb">sum</span> <span class="k">as</span> <span class="n">npsum</span><span class="p">,</span> <span class="n">asmatrix</span><span class="p">,</span> <span class="n">log</span><span class="p">,</span> <span class="n">asarray</span><span class="p">,</span> <span class="n">append</span>
-<span class="kn">from</span> <span class="nn">Helper</span> <span class="kn">import</span> <span class="n">transform1Dto2D</span>
+<span class="kn">from</span> <span class="nn">numpy</span> <span class="kn">import</span> <span class="nb">sum</span> <span class="k">as</span> <span class="n">npsum</span><span class="p">,</span> <span class="n">asmatrix</span><span class="p">,</span> <span class="n">log</span><span class="p">,</span> <span class="n">asarray</span><span class="p">,</span> <span class="n">append</span><span class="p">,</span> <span class="n">zeros</span><span class="p">,</span> <span class="n">reshape</span>
 
 <div class="viewcode-block" id="courseraML_CostFunc"><a class="viewcode-back" href="../costFunc.html#CostFunc.courseraML_CostFunc">[docs]</a><span class="k">def</span> <span class="nf">courseraML_CostFunc</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
     <span class="sd">r&#39;&#39;&#39;</span>
 
 <span class="sd">    .. math::</span>
 
-<span class="sd">        \frac{1}{m}&amp;\sum_{i=1}^{m}\sum_{k=1}^{K}[-y_k^i log(forwardPropThruAllLayers(x^i)_k)-(1-y_k^i) log(1-forwardPropThruAllLayers(x^i)_k)]\\</span>
-<span class="sd">        &amp;+\frac{\lambda}{2m}\sum^{allLayers} (weight^2~if~weight~is~NOT~multiplied~with~any~bias~unit~otherwise~0)\\</span>
+<span class="sd">        &amp;\frac{1}{m}\sum_{i=1}^{m}\sum_{k=1}^{K}[-y_k^i log(forwardThruAllLayers(x^i)_k)-(1-y_k^i) log(1-forwardThruAllLayers(x^i)_k)]\\</span>
+<span class="sd">        &amp;+\frac{\lambda}{2m}\sum^{allLayers~excludeAnyBias} (weight^2)\\</span>
 
-<span class="sd">    where m=the number of inputs; K=the number of labels in targets; y=a single target array; x=a single input array.</span>
+<span class="sd">    where :math:`m` =the number of inputs; :math:`k` =the number of labels in targets; :math:`y` =a single target array; :math:`x` =a single input array.</span>
 
-<span class="sd">    :param weightAllLayers: A flatten array contains all forward weights from input layer to the last hidden layer before output layer.</span>
+<span class="sd">    :param weightAllLayers: A flatten array contains all forward weights.</span>
 <span class="sd">    :param *args: Must in the following order:</span>
 
 <span class="sd">        **inputArr2D**: 1 training example per row.</span>
 
-<span class="sd">        **targets**: The number of labels must match the number of units in output layer.</span>
+<span class="sd">        **targets**: Must be a 2D ndarray instead of matrix. And the number of labels must match the number of units in output layer.</span>
 
 <span class="sd">        **weightDecayParam**: For model complexity regulation.</span>
 
 <span class="sd">    &#39;&#39;&#39;</span>
     <span class="n">inputArr2D</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">weightDecayParam</span><span class="p">,</span> <span class="n">nn</span> <span class="o">=</span> <span class="n">args</span>
     <span class="n">startIndex</span><span class="p">,</span> <span class="n">weightsExcBias</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span>
-    <span class="k">for</span> <span class="n">ly</span> <span class="ow">in</span> <span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">:</span> <span class="c"># update all forward weights</span>
-        <span class="n">newWeight</span> <span class="o">=</span> <span class="n">transform1Dto2D</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">[</span><span class="n">startIndex</span><span class="p">:</span><span class="n">startIndex</span> <span class="o">+</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">size</span><span class="p">],</span> <span class="o">*</span><span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
+    <span class="k">for</span> <span class="n">ly</span> <span class="ow">in</span> <span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">:</span>  <span class="c"># update all forward weights</span>
+        <span class="n">newWeight</span> <span class="o">=</span> <span class="n">reshape</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">[</span><span class="n">startIndex</span><span class="p">:</span><span class="n">startIndex</span> <span class="o">+</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">size</span><span class="p">],</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
         <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span> <span class="o">=</span> <span class="n">asmatrix</span><span class="p">(</span><span class="n">newWeight</span><span class="p">)</span>
         <span class="n">startIndex</span> <span class="o">+=</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">size</span>
-        <span class="n">weightsExcBias</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">weightsExcBias</span><span class="p">,</span> <span class="n">newWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> <span class="c"># exclude weights for bias unit with [:-1]</span>
+        <span class="n">weightsExcBias</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">weightsExcBias</span><span class="p">,</span> <span class="n">newWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>  <span class="c"># exclude weights for bias unit with [:-1]</span>
     <span class="n">output</span> <span class="o">=</span> <span class="n">asarray</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">forwardPropogateAllInput</span><span class="p">(</span><span class="n">inputArr2D</span><span class="p">))</span>
     <span class="k">assert</span> <span class="n">output</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="n">targets</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="s">&#39;dimension mismatch in next layer&#39;</span>
     <span class="k">return</span> <span class="mf">1.0</span> <span class="o">/</span> <span class="n">targets</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">*</span> <span class="p">(</span><span class="n">npsum</span><span class="p">(</span><span class="o">-</span><span class="n">targets</span> <span class="o">*</span> <span class="n">log</span><span class="p">(</span><span class="n">output</span><span class="p">)</span> <span class="o">-</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">targets</span><span class="p">)</span> <span class="o">*</span> <span class="n">log</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">output</span><span class="p">))</span> <span class="o">+</span> <span class="n">weightDecayParam</span> <span class="o">/</span> <span class="mf">2.0</span> <span class="o">*</span> <span class="n">npsum</span><span class="p">(</span><span class="n">weightsExcBias</span> <span class="o">**</span> <span class="mi">2</span><span class="p">))</span>
 </div>
 <div class="viewcode-block" id="courseraML_CostFuncGrad"><a class="viewcode-back" href="../costFunc.html#CostFunc.courseraML_CostFuncGrad">[docs]</a><span class="k">def</span> <span class="nf">courseraML_CostFuncGrad</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
     <span class="sd">r&#39;&#39;&#39;</span>
-<span class="sd">    Vectorized/regulated implementation that computes the partial derivatives of the :func:`courseraML_CostFunc` over each weight in weightAllLayers:</span>
+<span class="sd">    Vectorized/regulated implementation that computes the partial derivatives of the :func:`courseraML_CostFunc` over each weight (but exclude weights multiplied by bias units) in weightAllLayers:</span>
 
 <span class="sd">    .. math::</span>
 
 <span class="sd">        \frac{\partial~courseraMLCostFunc()}{\partial~weight_{ij}^{(layer~l)}}</span>
 
-<span class="sd">    :param weightAllLayers: A flatten array contains all forward weights from input layer to the last hidden layer before output layer.</span>
+<span class="sd">    :param weightAllLayers: A flatten array contains all forward weights.</span>
 <span class="sd">    :param *args: Must in the following order:</span>
 
 <span class="sd">        **inputArr2D**: 1 training example per row.</span>
 
 <span class="sd">    :returns: A flatten array represents the partial derivatives of the :func:`courseraML_CostFunc` over each weight in weightAllLayers.</span>
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="n">_</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">weightDecayParam</span><span class="p">,</span> <span class="n">nn</span> <span class="o">=</span> <span class="n">args</span> <span class="c"># no need to use weightAllLayers cause ly.forwardWeight will be updated in courseraML_CostFunc(), which will be used in FeedforwardNeuNet.train() together with courseraML_CostFuncGrad()</span>
-    <span class="n">costGradAllLyOutToIn</span> <span class="o">=</span> <span class="p">[]</span> <span class="c"># each is a triangle^{l}(a matrix) on Courera ML L9, p8</span>
+    <span class="n">_</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">weightDecayParam</span><span class="p">,</span> <span class="n">nn</span> <span class="o">=</span> <span class="n">args</span>  <span class="c"># no need to use weightAllLayers cause ly.forwardWeight will be updated in courseraML_CostFunc(), which will be used in FeedforwardNeuNet.train() together with courseraML_CostFuncGrad()</span>
+    <span class="n">costGradAllLyOutToIn</span> <span class="o">=</span> <span class="p">[]</span>  <span class="c"># each is a triangle^{l}(a matrix) on Courera ML L9, p8</span>
     <span class="n">numOfExamples</span> <span class="o">=</span> <span class="n">targets</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">*</span> <span class="mf">1.0</span>
     <span class="n">errDeltaMatNxtLv</span> <span class="o">=</span> <span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">outputs</span> <span class="o">-</span> <span class="n">targets</span><span class="p">)</span>
     <span class="k">for</span> <span class="n">ly</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">[</span><span class="mi">1</span><span class="p">:]):</span>
-        <span class="n">costGradAllLyOutToIn</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="o">.</span><span class="n">T</span> <span class="o">*</span> <span class="n">errDeltaMatNxtLv</span><span class="p">)</span> <span class="c"># Set m=(# of training examples), ly=layer below output layer =&gt; ly.self2D is a (m by (# of units, including bias)) matrix, and errDeltaMatNxtLv is a (m by (# of classes in output layer)) matrix. We originally have: outputLayerMatrix(which shares the same dimensions as errDeltaMatNxtLv)==ly.self2D*weightsMatrixBetweenOutputLyAndLyBelow(which shares the same dimensions as costGradAllLyOutToIn[n]), now we have: self.2D.T*outputLayerMatrix==self.2D.T*self.2D*weightsMatrixBetweenOutputLyAndLyBelow==weightsMatrixBetweenOutputLyAndLyBelow</span>
+        <span class="n">costGradAllLyOutToIn</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="o">.</span><span class="n">T</span> <span class="o">*</span> <span class="n">errDeltaMatNxtLv</span><span class="p">)</span>  <span class="c"># Set m=(# of training examples), ly=layer below output layer =&gt; ly.self2D is a (m by (# of units, including bias)) matrix, and errDeltaMatNxtLv is a (m by (# of classes in output layer)) matrix. We originally have: outputLayerMatrix(which shares the same dimensions as errDeltaMatNxtLv)==ly.self2D*weightsMatrixBetweenOutputLyAndLyBelow(which shares the same dimensions as costGradAllLyOutToIn[n]), now we have: self.2D.T*outputLayerMatrix==self.2D.T*self.2D*weightsMatrixBetweenOutputLyAndLyBelow==weightsMatrixBetweenOutputLyAndLyBelow</span>
         <span class="c"># (check sparseAutoencoder_2011new.pdf p8) cause all bias unit has value 1, so after the statement above, costGradAllLyOutToIn[-1][-1] == npsum(errDeltaMatNxtLv, 0)==the partial deri of bi at the botm of p7</span>
         <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">/</span> <span class="n">numOfExamples</span> <span class="o">*</span> <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
-        <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">weightDecayParam</span> <span class="o">/</span> <span class="n">numOfExamples</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="c"># add regularization but exclude weights for bias unit</span>
+        <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">weightDecayParam</span> <span class="o">/</span> <span class="n">numOfExamples</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>  <span class="c"># add regularization but exclude weights for bias unit</span>
         <span class="n">arr</span> <span class="o">=</span> <span class="n">asarray</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="p">)</span>
-        <span class="n">errDeltaMatNxtLv</span> <span class="o">=</span> <span class="n">asmatrix</span><span class="p">((</span><span class="n">asarray</span><span class="p">(</span><span class="n">errDeltaMatNxtLv</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">*</span> <span class="n">arr</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">arr</span><span class="p">))[:,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> <span class="c"># exclude bias unit</span>
-    <span class="n">ly</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="c"># no need to calculate errDeltaMatNxtLv for input layer, so separate it from for loop above</span>
-    <span class="n">costGradAllLyOutToIn</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="o">.</span><span class="n">T</span> <span class="o">*</span> <span class="n">errDeltaMatNxtLv</span><span class="p">)</span> <span class="c"># costGradAllLyOutToIn[-1] is the gradient of cost func over input layer&#39;s weights</span>
+        <span class="n">errDeltaMatNxtLv</span> <span class="o">=</span> <span class="n">asmatrix</span><span class="p">((</span><span class="n">asarray</span><span class="p">(</span><span class="n">errDeltaMatNxtLv</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">*</span> <span class="n">arr</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">arr</span><span class="p">))[:,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>  <span class="c"># exclude bias unit</span>
+    <span class="n">ly</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>  <span class="c"># no need to calculate errDeltaMatNxtLv for input layer, so separate it from for loop above</span>
+    <span class="n">costGradAllLyOutToIn</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="o">.</span><span class="n">T</span> <span class="o">*</span> <span class="n">errDeltaMatNxtLv</span><span class="p">)</span>  <span class="c"># costGradAllLyOutToIn[-1] is the gradient of cost func over input layer&#39;s weights</span>
     <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">/</span> <span class="n">numOfExamples</span> <span class="o">*</span> <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
     <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">weightDecayParam</span> <span class="o">/</span> <span class="n">numOfExamples</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
     <span class="n">flat</span> <span class="o">=</span> <span class="n">asarray</span><span class="p">(</span><span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
         <span class="n">flat</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">flat</span><span class="p">,</span> <span class="n">asarray</span><span class="p">(</span><span class="n">f</span><span class="p">))</span>
     <span class="k">return</span> <span class="n">flat</span>
 </div>
-<span class="k">def</span> <span class="nf">sparse_CostFunc</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
+<div class="viewcode-block" id="sparse_CostFunc"><a class="viewcode-back" href="../costFunc.html#CostFunc.sparse_CostFunc">[docs]</a><span class="k">def</span> <span class="nf">sparse_CostFunc</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
     <span class="sd">r&#39;&#39;&#39;</span>
-<span class="sd">    Vectorized/regulated cost function (described in the Coursera Stanford Machine Learning course) that computes the total cost over multiple inputs:</span>
+<span class="sd">    Vectorized/regulated sparse cost function (described in the sparseae_reading.pdf on the `UFLDL Tutorial Exercise:Sparse_Autoencoder &lt;http://ufldl.stanford.edu/wiki/index.php/Exercise:Sparse_Autoencoder&gt;`_) that computes the total cost over multiple inputs:</span>
 
 <span class="sd">    .. math::</span>
+<span class="sd">    </span>
+<span class="sd">        &amp;define: \hat{\rho}_j=\frac{1}{m}\sum_{i=1}^{m}[actvh_j(x^i)]\\</span>
+<span class="sd">        &amp;define: \sum_{j=1}^{h}KL(\rho||\hat{\rho}_j)=\sum_{j=1}^{f}\rho~log\frac{\rho}{\hat{\rho}_j}+(1-\rho)log\frac{1-\rho}{1-\hat{\rho}_j}\\</span>
+<span class="sd">        &amp;costFunction:~\frac{1}{m}\sum_{i=1}^{m}(0.5~||forwardThruAllLayers(x^i)-y^i||^2)]+\frac{\lambda}{2}\sum^{allLayers~excludeAnyBias} (weight^2)\\</span>
+<span class="sd">        &amp;+\beta\sum_{j=1}^{f}KL(\rho||\hat{\rho}_j)</span>
 
-<span class="sd">        \frac{1}{m}&amp;\sum_{i=1}^{m}\sum_{k=1}^{K}[-y_k^i log(forwardPropThruAllLayers(x^i)_k)-(1-y_k^i) log(1-forwardPropThruAllLayers(x^i)_k)]\\</span>
-<span class="sd">        &amp;+\frac{\lambda}{2m}\sum^{allLayers} (weight^2~if~weight~is~NOT~multiplied~with~any~bias~unit~otherwise~0)\\</span>
-
-<span class="sd">    where m=the number of inputs; K=the number of labels in targets; y=a single target array; x=a single input array.</span>
+<span class="sd">    where :math:`\hat{\rho}_j` =average activation of hidden unit j; :math:`m` =the number of inputs; :math:`h` =number of hidden units exclude bias units; :math:`y^i` =a single target array; :math:`x^i` =a single input array; :math:`\beta` =this is sparseParam.</span>
 
-<span class="sd">    :param weightAllLayers: A flatten array contains all forward weights from input layer to the last hidden layer before output layer.</span>
+<span class="sd">    :param weightAllLayers: A flatten array contains all forward weights.</span>
 <span class="sd">    :param *args: Must in the following order:</span>
 
 <span class="sd">        **inputArr2D**: 1 training example per row.</span>
 
-<span class="sd">        **targets**: The number of labels must match the number of units in output layer.</span>
+<span class="sd">        **targets**: Must be a 2D ndarray instead of matrix. And the number of labels must match the number of units in output layer.</span>
 
 <span class="sd">        **weightDecayParam**: For model complexity regulation.</span>
+<span class="sd">        </span>
+<span class="sd">        **sparsity**: Setting the sparsity of neural network.  </span>
+<span class="sd">        </span>
+<span class="sd">        **sparseParam**: For sparsity regulation.</span>
 
 <span class="sd">        **nn**: An instance of class FeedforwardNeuNet.</span>
 
     <span class="n">inputArr2D</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">weightDecayParam</span><span class="p">,</span> <span class="n">sparsity</span><span class="p">,</span> <span class="n">sparseParam</span><span class="p">,</span> <span class="n">nn</span> <span class="o">=</span> <span class="n">args</span>
     <span class="n">startIndex</span><span class="p">,</span> <span class="n">weightsExcBias</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span>
     <span class="n">avgEx</span> <span class="o">=</span> <span class="mf">1.0</span> <span class="o">/</span> <span class="n">targets</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
-    <span class="k">for</span> <span class="n">ly</span> <span class="ow">in</span> <span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">:</span> <span class="c"># update all forward weights</span>
-        <span class="n">newWeight</span> <span class="o">=</span> <span class="n">transform1Dto2D</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">[</span><span class="n">startIndex</span><span class="p">:</span><span class="n">startIndex</span> <span class="o">+</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">size</span><span class="p">],</span> <span class="o">*</span><span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
+    <span class="k">for</span> <span class="n">ly</span> <span class="ow">in</span> <span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">:</span>  <span class="c"># update all forward weights</span>
+        <span class="n">newWeight</span> <span class="o">=</span> <span class="n">reshape</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">[</span><span class="n">startIndex</span><span class="p">:</span><span class="n">startIndex</span> <span class="o">+</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">size</span><span class="p">],</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
         <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span> <span class="o">=</span> <span class="n">asmatrix</span><span class="p">(</span><span class="n">newWeight</span><span class="p">)</span>
         <span class="n">startIndex</span> <span class="o">+=</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">size</span>
-        <span class="n">weightsExcBias</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">weightsExcBias</span><span class="p">,</span> <span class="n">newWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> <span class="c"># exclude weights for bias unit with [:-1]</span>
+        <span class="n">weightsExcBias</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">weightsExcBias</span><span class="p">,</span> <span class="n">newWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>  <span class="c"># exclude weights for bias unit with [:-1]</span>
     <span class="n">output</span> <span class="o">=</span> <span class="n">asarray</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">forwardPropogateAllInput</span><span class="p">(</span><span class="n">inputArr2D</span><span class="p">))</span>
     <span class="k">assert</span> <span class="n">output</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">==</span> <span class="n">targets</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="s">&#39;dimension mismatch in next layer&#39;</span>
-<span class="c">#     avgActvArrAllLyAllEx = 0</span>
-<span class="c">#     for ly in nn.layersExOutputLy[1:]:</span>
-<span class="c">#         ly.avgActvArrAllEx = avgEx * npsum(ly.self2D, 0)</span>
-<span class="c">#         avgActvArrAllLyAllEx = append(avgActvArrAllLyAllEx, ly.avgActvArrAllEx[:-1]) # exclude bias?</span>
-    <span class="k">return</span> <span class="n">avgEx</span> <span class="o">*</span> <span class="p">(</span><span class="n">npsum</span><span class="p">(</span><span class="o">-</span><span class="n">targets</span> <span class="o">*</span> <span class="n">log</span><span class="p">(</span><span class="n">output</span><span class="p">)</span> <span class="o">-</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">targets</span><span class="p">)</span> <span class="o">*</span> <span class="n">log</span><span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">output</span><span class="p">))</span> <span class="o">+</span> <span class="n">weightDecayParam</span> <span class="o">/</span> <span class="mf">2.0</span> <span class="o">*</span> <span class="n">npsum</span><span class="p">(</span><span class="n">weightsExcBias</span> <span class="o">**</span> <span class="mi">2</span><span class="p">))</span> <span class="o">+</span> <span class="n">sparseParam</span> <span class="o">*</span> <span class="n">npsum</span><span class="p">(</span><span class="n">sparsity</span> <span class="o">*</span> <span class="n">log</span><span class="p">(</span><span class="n">sparsity</span> <span class="o">/</span> <span class="n">avgActvArrAllLyAllEx</span><span class="p">)</span> <span class="o">+</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">sparsity</span><span class="p">)</span> <span class="o">*</span> <span class="n">log</span><span class="p">((</span><span class="mi">1</span> <span class="o">-</span> <span class="n">sparsity</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">avgActvArrAllLyAllEx</span><span class="p">)))</span>
-
-<span class="k">def</span> <span class="nf">sparse_CostFuncGrad</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
+    <span class="n">avgActvArrAllLyAllEx</span> <span class="o">=</span> <span class="mi">0</span>
+    <span class="k">for</span> <span class="n">ly</span> <span class="ow">in</span> <span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">[</span><span class="mi">1</span><span class="p">:]:</span>
+        <span class="n">ly</span><span class="o">.</span><span class="n">avgActvArrAllEx</span> <span class="o">=</span> <span class="n">avgEx</span> <span class="o">*</span> <span class="n">npsum</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="p">[:,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="mi">0</span><span class="p">)</span>
+        <span class="n">avgActvArrAllLyAllEx</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">avgActvArrAllLyAllEx</span><span class="p">,</span> <span class="n">ly</span><span class="o">.</span><span class="n">avgActvArrAllEx</span><span class="p">)</span>  <span class="c"># not sure whether I should include bias here?</span>
+    <span class="n">avgActvArrAllLyAllEx</span> <span class="o">=</span> <span class="n">avgActvArrAllLyAllEx</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>  <span class="c"># discard 0 at the beginning</span>
+    <span class="k">return</span> <span class="n">avgEx</span> <span class="o">*</span> <span class="n">npsum</span><span class="p">(</span><span class="mf">0.5</span> <span class="o">*</span> <span class="p">(</span><span class="n">output</span> <span class="o">-</span> <span class="n">targets</span><span class="p">)</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">0.5</span> <span class="o">*</span> <span class="n">weightDecayParam</span> <span class="o">*</span> <span class="n">npsum</span><span class="p">(</span><span class="n">weightsExcBias</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="n">sparseParam</span> <span class="o">*</span> <span class="n">npsum</span><span class="p">(</span><span class="n">sparsity</span> <span class="o">*</span> <span class="n">log</span><span class="p">(</span><span class="n">sparsity</span> <span class="o">/</span> <span class="n">avgActvArrAllLyAllEx</span><span class="p">)</span> <span class="o">+</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">sparsity</span><span class="p">)</span> <span class="o">*</span> <span class="n">log</span><span class="p">((</span><span class="mi">1</span> <span class="o">-</span> <span class="n">sparsity</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">avgActvArrAllLyAllEx</span><span class="p">)))</span>
+    <span class="c"># return avgEx * (npsum(-targets * log(output) - (1 - targets) * log(1 - output)) + weightDecayParam / 2.0 * npsum(weightsExcBias ** 2)) + sparseParam * npsum(sparsity * log(sparsity / avgActvArrAllLyAllEx) + (1 - sparsity) * log((1 - sparsity) / (1 - avgActvArrAllLyAllEx)))</span>
+</div>
+<div class="viewcode-block" id="sparse_CostFuncGrad"><a class="viewcode-back" href="../costFunc.html#CostFunc.sparse_CostFuncGrad">[docs]</a><span class="k">def</span> <span class="nf">sparse_CostFuncGrad</span><span class="p">(</span><span class="n">weightAllLayers</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">):</span>
     <span class="sd">r&#39;&#39;&#39;</span>
-<span class="sd">    Vectorized/regulated implementation that computes the partial derivatives of the :func:`courseraML_CostFunc` over each weight in weightAllLayers:</span>
+<span class="sd">    Vectorized/regulated implementation that computes the partial derivatives of the :func:`sparse_CostFunc` over each weight (but exclude weights multiplied by bias units) in weightAllLayers:</span>
 
 <span class="sd">    .. math::</span>
 
-<span class="sd">        \frac{\partial~courseraMLCostFunc()}{\partial~weight_{ij}^{(layer~l)}}</span>
+<span class="sd">        \frac{\partial~sparseCostFunc()}{\partial~weight_{ij}^{(layer~l)}}</span>
 
-<span class="sd">    :param weightAllLayers: A flatten array contains all forward weights from input layer to the last hidden layer before output layer.</span>
+<span class="sd">    :param weightAllLayers: A flatten array contains all forward weights.</span>
 <span class="sd">    :param *args: Must in the following order:</span>
 
 <span class="sd">        **inputArr2D**: 1 training example per row.</span>
 
 <span class="sd">        **weightDecayParam**: For model complexity regulation.</span>
 
+<span class="sd">        **sparsity**: Setting the sparsity of neural network.  </span>
+<span class="sd">        </span>
+<span class="sd">        **sparseParam**: For sparsity regulation.</span>
+
 <span class="sd">        **nn**: An instance of class FeedforwardNeuNet.</span>
 
 <span class="sd">    :returns: A flatten array represents the partial derivatives of the :func:`courseraML_CostFunc` over each weight in weightAllLayers.</span>
 <span class="sd">    &#39;&#39;&#39;</span>
-    <span class="n">_</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">weightDecayParam</span><span class="p">,</span> <span class="n">nn</span> <span class="o">=</span> <span class="n">args</span> <span class="c"># no need to use weightAllLayers cause ly.forwardWeight will be updated in courseraML_CostFunc(), which will be used in FeedforwardNeuNet.train() together with courseraML_CostFuncGrad()</span>
-    <span class="n">costGradAllLyOutToIn</span> <span class="o">=</span> <span class="p">[]</span> <span class="c"># each is a triangle^{l}(a matrix) on Courera ML L9, p8</span>
+    <span class="n">_</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">weightDecayParam</span><span class="p">,</span> <span class="n">sparsity</span><span class="p">,</span> <span class="n">sparseParam</span><span class="p">,</span> <span class="n">nn</span> <span class="o">=</span> <span class="n">args</span>  <span class="c"># no need to use weightAllLayers cause ly.forwardWeight will be updated in courseraML_CostFunc(), which will be used in FeedforwardNeuNet.train() together with courseraML_CostFuncGrad()</span>
+    <span class="n">costGradAllLyOutToIn</span> <span class="o">=</span> <span class="p">[]</span>  <span class="c"># each is a triangle^{l}(a matrix) on Courera ML L9, p8</span>
     <span class="n">numOfExamples</span> <span class="o">=</span> <span class="n">targets</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">*</span> <span class="mf">1.0</span>
-    <span class="n">errDeltaMatNxtLv</span> <span class="o">=</span> <span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">outputs</span> <span class="o">-</span> <span class="n">targets</span><span class="p">)</span>
+    <span class="n">arr</span> <span class="o">=</span> <span class="n">asarray</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">outputs</span><span class="p">)</span>
+    <span class="n">errDeltaMatNxtLv</span> <span class="o">=</span> <span class="n">asmatrix</span><span class="p">(</span><span class="n">asarray</span><span class="p">((</span><span class="n">nn</span><span class="o">.</span><span class="n">outputs</span> <span class="o">-</span> <span class="n">targets</span><span class="p">))</span> <span class="o">*</span> <span class="n">arr</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">arr</span><span class="p">))</span>
     <span class="k">for</span> <span class="n">ly</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">[</span><span class="mi">1</span><span class="p">:]):</span>
-        <span class="n">costGradAllLyOutToIn</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="o">.</span><span class="n">T</span> <span class="o">*</span> <span class="n">errDeltaMatNxtLv</span><span class="p">)</span> <span class="c"># Set m=(# of training examples), ly=layer below output layer =&gt; ly.self2D is a (m by (# of units, including bias)) matrix, and errDeltaMatNxtLv is a (m by (# of classes in output layer)) matrix. We originally have: outputLayerMatrix(which shares the same dimensions as errDeltaMatNxtLv)==ly.self2D*weightsMatrixBetweenOutputLyAndLyBelow(which shares the same dimensions as costGradAllLyOutToIn[n]), now we have: self.2D.T*outputLayerMatrix==self.2D.T*self.2D*weightsMatrixBetweenOutputLyAndLyBelow==weightsMatrixBetweenOutputLyAndLyBelow</span>
+        <span class="n">costGradAllLyOutToIn</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="o">.</span><span class="n">T</span> <span class="o">*</span> <span class="n">errDeltaMatNxtLv</span><span class="p">)</span>  <span class="c"># Set m=(# of training examples), ly=layer below output layer =&gt; ly.self2D is a (m by (# of units, including bias)) matrix, and errDeltaMatNxtLv is a (m by (# of classes in output layer)) matrix. We originally have: outputLayerMatrix(which shares the same dimensions as errDeltaMatNxtLv)==ly.self2D*weightsMatrixBetweenOutputLyAndLyBelow(which shares the same dimensions as costGradAllLyOutToIn[n]), now we have: self.2D.T*outputLayerMatrix==self.2D.T*self.2D*weightsMatrixBetweenOutputLyAndLyBelow==weightsMatrixBetweenOutputLyAndLyBelow</span>
         <span class="c"># (check sparseAutoencoder_2011new.pdf p8) cause all bias unit has value 1, so after the statement above, costGradAllLyOutToIn[-1][-1] == npsum(errDeltaMatNxtLv, 0)==the partial deri of bi at the botm of p7</span>
         <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">/</span> <span class="n">numOfExamples</span> <span class="o">*</span> <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
-        <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">weightDecayParam</span> <span class="o">/</span> <span class="n">numOfExamples</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="c"># add regularization but exclude weights for bias unit</span>
+        <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">weightDecayParam</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>  <span class="c"># add regularization but exclude weights for bias unit</span>
         <span class="n">arr</span> <span class="o">=</span> <span class="n">asarray</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="p">)</span>
-        <span class="n">errDeltaMatNxtLv</span> <span class="o">=</span> <span class="n">asmatrix</span><span class="p">((</span><span class="n">asarray</span><span class="p">(</span><span class="n">errDeltaMatNxtLv</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">T</span><span class="p">)</span> <span class="o">*</span> <span class="n">arr</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">arr</span><span class="p">))[:,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span> <span class="c"># exclude bias unit</span>
-    <span class="n">ly</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="c"># no need to calculate errDeltaMatNxtLv for input layer, so separate it from for loop above</span>
-    <span class="n">costGradAllLyOutToIn</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="o">.</span><span class="n">T</span> <span class="o">*</span> <span class="n">errDeltaMatNxtLv</span><span class="p">)</span> <span class="c"># costGradAllLyOutToIn[-1] is the gradient of cost func over input layer&#39;s weights</span>
+        <span class="n">s</span> <span class="o">=</span> <span class="n">sparseParam</span> <span class="o">*</span> <span class="p">(</span><span class="o">-</span><span class="n">sparsity</span> <span class="o">/</span> <span class="n">ly</span><span class="o">.</span><span class="n">avgActvArrAllEx</span> <span class="o">+</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">sparsity</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">ly</span><span class="o">.</span><span class="n">avgActvArrAllEx</span><span class="p">))</span>
+        <span class="n">s</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="n">zeros</span><span class="p">((</span><span class="n">s</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">1</span><span class="p">)),</span> <span class="mi">1</span><span class="p">)</span>  <span class="c"># append zeros beca instead of getting rid of bias units for every term(errDeltaMatNxtLv * ly.forwardWeight.T, s, arr in the following statement), I decided to dump them as the final step</span>
+        <span class="n">errDeltaMatNxtLv</span> <span class="o">=</span> <span class="n">asmatrix</span><span class="p">(</span><span class="n">asarray</span><span class="p">(</span><span class="n">errDeltaMatNxtLv</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="o">.</span><span class="n">T</span> <span class="o">+</span> <span class="n">s</span><span class="p">)</span> <span class="o">*</span> <span class="n">arr</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">arr</span><span class="p">))[:,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>  <span class="c"># exclude bias unit</span>
+    <span class="n">ly</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>  <span class="c"># no need to calculate errDeltaMatNxtLv for input layer, so separate it from for loop above</span>
+    <span class="n">costGradAllLyOutToIn</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="o">.</span><span class="n">T</span> <span class="o">*</span> <span class="n">errDeltaMatNxtLv</span><span class="p">)</span>  <span class="c"># costGradAllLyOutToIn[-1] is the gradient of cost func over input layer&#39;s weights</span>
     <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">/</span> <span class="n">numOfExamples</span> <span class="o">*</span> <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
     <span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">][:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span> <span class="o">+=</span> <span class="n">weightDecayParam</span> <span class="o">/</span> <span class="n">numOfExamples</span> <span class="o">*</span> <span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
     <span class="n">flat</span> <span class="o">=</span> <span class="n">asarray</span><span class="p">(</span><span class="n">costGradAllLyOutToIn</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">])</span>
     <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="nb">reversed</span><span class="p">(</span><span class="n">costGradAllLyOutToIn</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]):</span>
         <span class="n">flat</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">flat</span><span class="p">,</span> <span class="n">asarray</span><span class="p">(</span><span class="n">f</span><span class="p">))</span>
-    <span class="k">return</span> <span class="n">flat</span>
+    <span class="k">return</span> <span class="n">flat</span></div>
 </pre></div>
 
           </div>
         <li class="right" style="margin-right: 10px">
           <a href="../genindex.html" title="General Index"
              >index</a></li>
-        <li class="right" >
-          <a href="../py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="../index.html">deepArch 0.1 documentation</a> &raquo;</li>
           <li><a href="index.html" >Module code</a> &raquo;</li> 
       </ul>

File deepArch/_modules/FeedforwardNeuNet.html

     <script type="text/javascript" src="../_static/jquery.js"></script>
     <script type="text/javascript" src="../_static/underscore.js"></script>
     <script type="text/javascript" src="../_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="../index.html" />
     <link rel="up" title="Module code" href="index.html" /> 
   </head>
         <li class="right" style="margin-right: 10px">
           <a href="../genindex.html" title="General Index"
              accesskey="I">index</a></li>
-        <li class="right" >
-          <a href="../py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="../index.html">deepArch 0.1 documentation</a> &raquo;</li>
           <li><a href="index.html" accesskey="U">Module code</a> &raquo;</li> 
       </ul>
 <span class="sd">        &gt;&gt;&gt; layer</span>
 <span class="sd">        NnLayer([0.23, 0.37, 0.28, 1.])</span>
 <span class="sd">        &#39;&#39;&#39;</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">self2D</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">inputArr2D</span><span class="p">,</span> <span class="n">ones</span><span class="p">((</span><span class="n">inputArr2D</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">1</span><span class="p">))</span> <span class="o">*</span> <span class="bp">self</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="mi">1</span><span class="p">)</span> <span class="c"># self2D[-1] == bias</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">self2D</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">inputArr2D</span><span class="p">,</span> <span class="n">ones</span><span class="p">((</span><span class="n">inputArr2D</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">1</span><span class="p">))</span> <span class="o">*</span> <span class="bp">self</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="mi">1</span><span class="p">)</span>  <span class="c"># self2D[-1] == bias</span>
         <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">__activFunc</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">self2D</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">forwardWeight</span><span class="p">)</span>
 </div>
 <div class="viewcode-block" id="NnLayer.updateForwardWeight"><a class="viewcode-back" href="../deepArchitecture.html#FeedforwardNeuNet.NnLayer.updateForwardWeight">[docs]</a>    <span class="k">def</span> <span class="nf">updateForwardWeight</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">newWeight</span><span class="p">):</span>
         <span class="sd">&#39;&#39;&#39;</span>
 <span class="sd">        :param layersExOutputLy: A tuple that contains all layers except the output layer.</span>
 <span class="sd">        :param weightDecayParam: Parameter that regulates network model complexity.</span>
-<span class="sd">        :param sparsity: Parameter that set the threshold of the sparsity of the neural network.</span>
+<span class="sd">        :param sparsity: Parameter that sets the target value of the sparsity of the neural network.</span>
 <span class="sd">        :param sparseParam: Parameter that regulates the sparsity.</span>
 
 <span class="sd">        &gt;&gt;&gt; layer0, layer1 = NnLayer(sigmoid, 3, 1, 2), NnLayer(sigmoid, 2, 1, 6)</span>
 <span class="sd">        :param costFunc: callable *f(paramToOptimize, \*arg)* that will be used as cost function.</span>
 <span class="sd">        :param costFuncGrad: callable *f&#39;(paramToOptimize, \*arg)* that will be used to compute partial derivative of cost function over each parameter in paramToOptimize.</span>
 <span class="sd">        &#39;&#39;&#39;</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">forwardPropogateAllInput</span><span class="p">(</span><span class="n">inputArr2D</span><span class="p">)</span> <span class="c"># perform forward propagation to set self.outputs</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">forwardPropogateAllInput</span><span class="p">(</span><span class="n">inputArr2D</span><span class="p">)</span>  <span class="c"># perform forward propagation to set self.outputs</span>
         <span class="n">avgEx</span> <span class="o">=</span> <span class="mf">1.0</span> <span class="o">/</span> <span class="n">targets</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
         <span class="n">flatWeights</span> <span class="o">=</span> <span class="n">asarray</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">.</span><span class="n">forwardWeight</span><span class="p">)</span>
         <span class="k">for</span> <span class="n">ly</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">layersExOutputLy</span><span class="p">[</span><span class="mi">1</span><span class="p">:]:</span>
-            <span class="n">ly</span><span class="o">.</span><span class="n">avgActvArrAllEx</span> <span class="o">=</span> <span class="n">avgEx</span> <span class="o">*</span> <span class="n">npsum</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
+            <span class="n">ly</span><span class="o">.</span><span class="n">avgActvArrAllEx</span> <span class="o">=</span> <span class="n">avgEx</span> <span class="o">*</span> <span class="n">npsum</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">self2D</span><span class="p">[:,</span> <span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="mi">0</span><span class="p">)</span>
             <span class="n">flatWeights</span> <span class="o">=</span> <span class="n">append</span><span class="p">(</span><span class="n">flatWeights</span><span class="p">,</span> <span class="n">asarray</span><span class="p">(</span><span class="n">ly</span><span class="o">.</span><span class="n">forwardWeight</span><span class="p">))</span>
-        <span class="n">fmin_cg</span><span class="p">(</span><span class="n">costFunc</span><span class="p">,</span> <span class="n">flatWeights</span><span class="p">,</span> <span class="n">costFuncGrad</span><span class="p">,</span> <span class="p">(</span><span class="n">inputArr2D</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__weightDecayParam</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__sparsity</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__sparseParam</span><span class="p">,</span> <span class="bp">self</span><span class="p">),</span> <span class="n">maxiter</span><span class="o">=</span><span class="n">maxIter</span><span class="p">,</span> <span class="n">full_output</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span> <span class="c"># fmin_cg calls grad before cost func</span>
-<span class="c">#         startIndex = 0 # update all forward weights(not really sure this is necessary cause I already done this in cost func)</span>
-<span class="c">#         for ly in self.layersExOutputLy:</span>
-<span class="c">#             newWeight = transform1Dto2D(optForwardWeights[startIndex:startIndex + ly.forwardWeight.size], *ly.forwardWeight.shape)</span>
-<span class="c">#             ly.forwardWeight = asmatrix(newWeight)</span>
-<span class="c">#             startIndex += ly.forwardWeight.size</span>
+        <span class="n">fmin_cg</span><span class="p">(</span><span class="n">costFunc</span><span class="p">,</span> <span class="n">flatWeights</span><span class="p">,</span> <span class="n">costFuncGrad</span><span class="p">,</span> <span class="p">(</span><span class="n">inputArr2D</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__weightDecayParam</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__sparsity</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">__sparseParam</span><span class="p">,</span> <span class="bp">self</span><span class="p">),</span> <span class="n">maxiter</span><span class="o">=</span><span class="n">maxIter</span><span class="p">,</span> <span class="n">full_output</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>  <span class="c"># fmin_cg calls grad before cost func</span>
 </div>
 <div class="viewcode-block" id="FeedforwardNeuNet.forwardPropogateOneInput"><a class="viewcode-back" href="../deepArchitecture.html#FeedforwardNeuNet.FeedforwardNeuNet.forwardPropogateOneInput">[docs]</a>    <span class="k">def</span> <span class="nf">forwardPropogateOneInput</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputArr</span><span class="p">):</span>
         <span class="sd">&#39;&#39;&#39;</span>
         <li class="right" style="margin-right: 10px">
           <a href="../genindex.html" title="General Index"
              >index</a></li>
-        <li class="right" >
-          <a href="../py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="../index.html">deepArch 0.1 documentation</a> &raquo;</li>
           <li><a href="index.html" >Module code</a> &raquo;</li> 
       </ul>

File deepArch/_modules/index.html

     <script type="text/javascript" src="../_static/jquery.js"></script>
     <script type="text/javascript" src="../_static/underscore.js"></script>
     <script type="text/javascript" src="../_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="../index.html" /> 
   </head>
   <body>
         <li class="right" style="margin-right: 10px">
           <a href="../genindex.html" title="General Index"
              accesskey="I">index</a></li>
-        <li class="right" >
-          <a href="../py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="../index.html">deepArch 0.1 documentation</a> &raquo;</li> 
       </ul>
     </div>  
         <li class="right" style="margin-right: 10px">
           <a href="../genindex.html" title="General Index"
              >index</a></li>
-        <li class="right" >
-          <a href="../py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="../index.html">deepArch 0.1 documentation</a> &raquo;</li> 
       </ul>
     </div>

File deepArch/_sources/costFunc.txt

 
 Stanford Coursera Machine Learning course
 -----------------------------------------
-.. automodule:: CostFunc
-	:members: courseraML_CostFunc, courseraML_CostFuncGrad
+.. currentmodule:: CostFunc
+.. autofunction:: courseraML_CostFunc
+.. autofunction:: courseraML_CostFuncGrad
+
+Stanford UFLDL Tutorial
+-----------------------
+.. autofunction:: sparse_CostFunc
+.. autofunction:: sparse_CostFuncGrad
 
 References
-++++++++++
+----------
 1. sec. 11.5 *Multilayer Perceptrons*, 11.7.1 *Backpropagation Algorithm*, `Introduction to Machine Learning, 2nd edition, Ethem Alpaydin <http://mitpress.mit.edu/books/introduction-machine-learning>`_.
 2. Lecture 9 *Neural Networks: Learning*, `Coursera Stanford Machine Learning course <https://www.coursera.org/course/ml>`_.
-3. http://ufldl.stanford.edu/wiki/index.php/UFLDL_Tutorial 
+3. http://ufldl.stanford.edu/wiki/index.php/Exercise:Sparse_Autoencoder 

File deepArch/_sources/deepArchitecture.txt

 
 Feedforward Neural Network Structure
 ------------------------------------
-.. image:: https://lh6.googleusercontent.com/-h9JB8KqOcK8/UfI_lmuSTwI/AAAAAAAAGxg/DjrkXas5u5c/w926-h522-no/FeedForwardNNStruct.png
+.. image:: https://lh4.googleusercontent.com/-zt-5emltAGw/Ufp9V81KYlI/AAAAAAAAHAE/BUtwvYS9wHU/w833-h467-no/FeedforwardNNStruct.png
 
 .. |addBias| image:: https://lh3.googleusercontent.com/-ZoJswguVmxQ/UeZHe6Y8zeI/AAAAAAAAGpk/RDza9WEoHzk/w48-h44-no/addBias.png
 	:width: 40px

File deepArch/_sources/demo.txt

 
 Feedforward Neural Network
 --------------------------
-I use the data set in `Digit Recognizer <http://www.kaggle.com/c/digit-recognizer>`_ competition on Kaggle, and achieved 97.371% accuracy on public test set. You can check out the demonstration in ipython notebook through ipython notebook viewer `here <http://nbviewer.ipython.org/5990038>`_.
+I use the data set in `Digit Recognizer <http://www.kaggle.com/c/digit-recognizer>`_ competition on Kaggle, and achieved 97.371% accuracy on public test set. You can check out the demonstration in ipython notebook through ipython nbviewer `here <http://nbviewer.ipython.org/5990038>`_.
+
+Sparse Autoencoder
+------------------
+I use the IMAGE.mat data set in `sparseae_exercise.zip on UFLDL Tutorial Exercise:Sparse_Autoencoder <http://ufldl.stanford.edu/wiki/index.php/Exercise:Sparse_Autoencoder>`_. You can check out the demonstration in ipython notebook through ipython nbviewer `here <http://nbviewer.ipython.org/6132034>`_.

File deepArch/costFunc.html

     <script type="text/javascript" src="_static/jquery.js"></script>
     <script type="text/javascript" src="_static/underscore.js"></script>
     <script type="text/javascript" src="_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="index.html" />
     <link rel="next" title="4. Demonstrations" href="demo.html" />
     <link rel="prev" title="2. Deep Architectures" href="deepArchitecture.html" /> 
           <a href="genindex.html" title="General Index"
              accesskey="I">index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="demo.html" title="4. Demonstrations"
              accesskey="N">next</a> |</li>
         <li class="right" >
   <div class="section" id="cost-functions">
 <span id="costfunc"></span><h1>3. Cost Functions<a class="headerlink" href="#cost-functions" title="Permalink to this headline">¶</a></h1>
 <p>This module contains cost functions and corresponding functions for computing derivatives.</p>
-<div class="section" id="module-CostFunc">
-<span id="stanford-coursera-machine-learning-course"></span><h2>3.1. Stanford Coursera Machine Learning course<a class="headerlink" href="#module-CostFunc" title="Permalink to this headline">¶</a></h2>
+<div class="section" id="stanford-coursera-machine-learning-course">
+<h2>3.1. Stanford Coursera Machine Learning course<a class="headerlink" href="#stanford-coursera-machine-learning-course" title="Permalink to this headline">¶</a></h2>
 <dl class="function">
 <dt id="CostFunc.courseraML_CostFunc">
 <tt class="descclassname">CostFunc.</tt><tt class="descname">courseraML_CostFunc</tt><big>(</big><em>weightAllLayers</em>, <em>*args</em><big>)</big><a class="reference internal" href="_modules/CostFunc.html#courseraML_CostFunc"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#CostFunc.courseraML_CostFunc" title="Permalink to this definition">¶</a></dt>
 <dd><p>Vectorized/regulated cost function (described in the Coursera Stanford Machine Learning course) that computes the total cost over multiple inputs:</p>
 <div class="math">
-<p><img src="_images/math/6e00b17315c0ef793e7fea24014ae1f68a0387f8.png" alt="\frac{1}{m}&amp;\sum_{i=1}^{m}\sum_{k=1}^{K}[-y_k^i log(forwardPropThruAllLayers(x^i)_k)-(1-y_k^i) log(1-forwardPropThruAllLayers(x^i)_k)]\\
-&amp;+\frac{\lambda}{2m}\sum^{allLayers} (weight^2~if~weight~is~NOT~multiplied~with~any~bias~unit~otherwise~0)\\"/></p>
-</div><p>where m=the number of inputs; K=the number of labels in targets; y=a single target array; x=a single input array.</p>
+\[\begin{split}&amp;\frac{1}{m}\sum_{i=1}^{m}\sum_{k=1}^{K}[-y_k^i log(forwardThruAllLayers(x^i)_k)-(1-y_k^i) log(1-forwardThruAllLayers(x^i)_k)]\\
+&amp;+\frac{\lambda}{2m}\sum^{allLayers~excludeAnyBias} (weight^2)\\\end{split}\]</div>
+<p>where <span class="math">\(m\)</span> =the number of inputs; <span class="math">\(k\)</span> =the number of labels in targets; <span class="math">\(y\)</span> =a single target array; <span class="math">\(x\)</span> =a single input array.</p>
 <table class="docutils field-list" frame="void" rules="none">
 <col class="field-name" />
 <col class="field-body" />
 <tbody valign="top">
 <tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
-<li><strong>weightAllLayers</strong> &#8211; A flatten array contains all forward weights from input layer to the last hidden layer before output layer.</li>
+<li><strong>weightAllLayers</strong> &#8211; A flatten array contains all forward weights.</li>
 <li><strong>*args</strong> &#8211; <p>Must in the following order:</p>
 <p><strong>inputArr2D</strong>: 1 training example per row.</p>
-<p><strong>targets</strong>: The number of labels must match the number of units in output layer.</p>
+<p><strong>targets</strong>: Must be a 2D ndarray instead of matrix. And the number of labels must match the number of units in output layer.</p>
 <p><strong>weightDecayParam</strong>: For model complexity regulation.</p>
 <p><strong>nn</strong>: An instance of class FeedforwardNeuNet.</p>
 </li>
 <dl class="function">
 <dt id="CostFunc.courseraML_CostFuncGrad">
 <tt class="descclassname">CostFunc.</tt><tt class="descname">courseraML_CostFuncGrad</tt><big>(</big><em>weightAllLayers</em>, <em>*args</em><big>)</big><a class="reference internal" href="_modules/CostFunc.html#courseraML_CostFuncGrad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#CostFunc.courseraML_CostFuncGrad" title="Permalink to this definition">¶</a></dt>
-<dd><p>Vectorized/regulated implementation that computes the partial derivatives of the <a class="reference internal" href="#CostFunc.courseraML_CostFunc" title="CostFunc.courseraML_CostFunc"><tt class="xref py py-func docutils literal"><span class="pre">courseraML_CostFunc()</span></tt></a> over each weight in weightAllLayers:</p>
+<dd><p>Vectorized/regulated implementation that computes the partial derivatives of the <a class="reference internal" href="#CostFunc.courseraML_CostFunc" title="CostFunc.courseraML_CostFunc"><tt class="xref py py-func docutils literal"><span class="pre">courseraML_CostFunc()</span></tt></a> over each weight (but exclude weights multiplied by bias units) in weightAllLayers:</p>
+<div class="math">
+\[\frac{\partial~courseraMLCostFunc()}{\partial~weight_{ij}^{(layer~l)}}\]</div>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
+<li><strong>weightAllLayers</strong> &#8211; A flatten array contains all forward weights.</li>
+<li><strong>*args</strong> &#8211; <p>Must in the following order:</p>
+<p><strong>inputArr2D</strong>: 1 training example per row.</p>
+<p><strong>targets</strong>: The number of labels must match the number of units in output layer.</p>
+<p><strong>weightDecayParam</strong>: For model complexity regulation.</p>
+<p><strong>nn</strong>: An instance of class FeedforwardNeuNet.</p>
+</li>
+</ul>
+</td>
+</tr>
+<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first last">A flatten array represents the partial derivatives of the <a class="reference internal" href="#CostFunc.courseraML_CostFunc" title="CostFunc.courseraML_CostFunc"><tt class="xref py py-func docutils literal"><span class="pre">courseraML_CostFunc()</span></tt></a> over each weight in weightAllLayers.</p>
+</td>
+</tr>
+</tbody>
+</table>
+</dd></dl>
+
+</div>
+<div class="section" id="stanford-ufldl-tutorial">
+<h2>3.2. Stanford UFLDL Tutorial<a class="headerlink" href="#stanford-ufldl-tutorial" title="Permalink to this headline">¶</a></h2>
+<dl class="function">
+<dt id="CostFunc.sparse_CostFunc">
+<tt class="descclassname">CostFunc.</tt><tt class="descname">sparse_CostFunc</tt><big>(</big><em>weightAllLayers</em>, <em>*args</em><big>)</big><a class="reference internal" href="_modules/CostFunc.html#sparse_CostFunc"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#CostFunc.sparse_CostFunc" title="Permalink to this definition">¶</a></dt>
+<dd><p>Vectorized/regulated sparse cost function (described in the sparseae_reading.pdf on the <a class="reference external" href="http://ufldl.stanford.edu/wiki/index.php/Exercise:Sparse_Autoencoder">UFLDL Tutorial Exercise:Sparse_Autoencoder</a>) that computes the total cost over multiple inputs:</p>
+<div class="math">
+\[\begin{split}&amp;define: \hat{\rho}_j=\frac{1}{m}\sum_{i=1}^{m}[actvh_j(x^i)]\\
+&amp;define: \sum_{j=1}^{h}KL(\rho||\hat{\rho}_j)=\sum_{j=1}^{f}\rho~log\frac{\rho}{\hat{\rho}_j}+(1-\rho)log\frac{1-\rho}{1-\hat{\rho}_j}\\
+&amp;costFunction:~\frac{1}{m}\sum_{i=1}^{m}(0.5~||forwardThruAllLayers(x^i)-y^i||^2)]+\frac{\lambda}{2}\sum^{allLayers~excludeAnyBias} (weight^2)\\
+&amp;+\beta\sum_{j=1}^{f}KL(\rho||\hat{\rho}_j)\end{split}\]</div>
+<p>where <span class="math">\(\hat{\rho}_j\)</span> =average activation of hidden unit j; <span class="math">\(m\)</span> =the number of inputs; <span class="math">\(h\)</span> =number of hidden units exclude bias units; <span class="math">\(y^i\)</span> =a single target array; <span class="math">\(x^i\)</span> =a single input array; <span class="math">\(\beta\)</span> =this is sparseParam.</p>
+<table class="docutils field-list" frame="void" rules="none">
+<col class="field-name" />
+<col class="field-body" />
+<tbody valign="top">
+<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
+<li><strong>weightAllLayers</strong> &#8211; A flatten array contains all forward weights.</li>
+<li><strong>*args</strong> &#8211; <p>Must in the following order:</p>
+<p><strong>inputArr2D</strong>: 1 training example per row.</p>
+<p><strong>targets</strong>: Must be a 2D ndarray instead of matrix. And the number of labels must match the number of units in output layer.</p>
+<p><strong>weightDecayParam</strong>: For model complexity regulation.</p>
+<p><strong>sparsity</strong>: Setting the sparsity of neural network.</p>
+<p><strong>sparseParam</strong>: For sparsity regulation.</p>
+<p><strong>nn</strong>: An instance of class FeedforwardNeuNet.</p>
+</li>
+</ul>
+</td>
+</tr>
+<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first last">A scalar representing the cost of current input using weightAllLayers.</p>
+</td>
+</tr>
+</tbody>
+</table>
+</dd></dl>
+
+<dl class="function">
+<dt id="CostFunc.sparse_CostFuncGrad">
+<tt class="descclassname">CostFunc.</tt><tt class="descname">sparse_CostFuncGrad</tt><big>(</big><em>weightAllLayers</em>, <em>*args</em><big>)</big><a class="reference internal" href="_modules/CostFunc.html#sparse_CostFuncGrad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#CostFunc.sparse_CostFuncGrad" title="Permalink to this definition">¶</a></dt>
+<dd><p>Vectorized/regulated implementation that computes the partial derivatives of the <a class="reference internal" href="#CostFunc.sparse_CostFunc" title="CostFunc.sparse_CostFunc"><tt class="xref py py-func docutils literal"><span class="pre">sparse_CostFunc()</span></tt></a> over each weight (but exclude weights multiplied by bias units) in weightAllLayers:</p>
 <div class="math">
-<p><img src="_images/math/25c8a77c4942de9aa1a7a923c9a77e2dde90f076.png" alt="\frac{\partial~courseraMLCostFunc()}{\partial~weight_{ij}^{(layer~l)}}"/></p>
-</div><table class="docutils field-list" frame="void" rules="none">
+\[\frac{\partial~sparseCostFunc()}{\partial~weight_{ij}^{(layer~l)}}\]</div>
+<table class="docutils field-list" frame="void" rules="none">
 <col class="field-name" />
 <col class="field-body" />
 <tbody valign="top">
 <tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first simple">
-<li><strong>weightAllLayers</strong> &#8211; A flatten array contains all forward weights from input layer to the last hidden layer before output layer.</li>
+<li><strong>weightAllLayers</strong> &#8211; A flatten array contains all forward weights.</li>
 <li><strong>*args</strong> &#8211; <p>Must in the following order:</p>
 <p><strong>inputArr2D</strong>: 1 training example per row.</p>
 <p><strong>targets</strong>: The number of labels must match the number of units in output layer.</p>
 <p><strong>weightDecayParam</strong>: For model complexity regulation.</p>
+<p><strong>sparsity</strong>: Setting the sparsity of neural network.</p>
+<p><strong>sparseParam</strong>: For sparsity regulation.</p>
 <p><strong>nn</strong>: An instance of class FeedforwardNeuNet.</p>
 </li>
 </ul>
 </table>
 </dd></dl>
 
+</div>
 <div class="section" id="references">
-<h3>3.1.1. References<a class="headerlink" href="#references" title="Permalink to this headline">¶</a></h3>
+<h2>3.3. References<a class="headerlink" href="#references" title="Permalink to this headline">¶</a></h2>
 <ol class="arabic simple">
 <li>sec. 11.5 <em>Multilayer Perceptrons</em>, 11.7.1 <em>Backpropagation Algorithm</em>, <a class="reference external" href="http://mitpress.mit.edu/books/introduction-machine-learning">Introduction to Machine Learning, 2nd edition, Ethem Alpaydin</a>.</li>
 <li>Lecture 9 <em>Neural Networks: Learning</em>, <a class="reference external" href="https://www.coursera.org/course/ml">Coursera Stanford Machine Learning course</a>.</li>
-<li><a class="reference external" href="http://ufldl.stanford.edu/wiki/index.php/UFLDL_Tutorial">http://ufldl.stanford.edu/wiki/index.php/UFLDL_Tutorial</a></li>
+<li><a class="reference external" href="http://ufldl.stanford.edu/wiki/index.php/Exercise:Sparse_Autoencoder">http://ufldl.stanford.edu/wiki/index.php/Exercise:Sparse_Autoencoder</a></li>
 </ol>
 </div>
 </div>
-</div>
 
 
           </div>
   <h3><a href="index.html">Table Of Contents</a></h3>
   <ul>
 <li><a class="reference internal" href="#">3. Cost Functions</a><ul>
-<li><a class="reference internal" href="#module-CostFunc">3.1. Stanford Coursera Machine Learning course</a><ul>
-<li><a class="reference internal" href="#references">3.1.1. References</a></li>
-</ul>
-</li>
+<li><a class="reference internal" href="#stanford-coursera-machine-learning-course">3.1. Stanford Coursera Machine Learning course</a></li>
+<li><a class="reference internal" href="#stanford-ufldl-tutorial">3.2. Stanford UFLDL Tutorial</a></li>
+<li><a class="reference internal" href="#references">3.3. References</a></li>
 </ul>
 </li>
 </ul>
           <a href="genindex.html" title="General Index"
              >index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="demo.html" title="4. Demonstrations"
              >next</a> |</li>
         <li class="right" >

File deepArch/deepArchitecture.html

     <script type="text/javascript" src="_static/jquery.js"></script>
     <script type="text/javascript" src="_static/underscore.js"></script>
     <script type="text/javascript" src="_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="index.html" />
     <link rel="next" title="3. Cost Functions" href="costFunc.html" />
     <link rel="prev" title="1. Introduction" href="introduction.html" /> 
           <a href="genindex.html" title="General Index"
              accesskey="I">index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="costFunc.html" title="3. Cost Functions"
              accesskey="N">next</a> |</li>
         <li class="right" >
 <p>Various types of deep architectures.</p>
 <div class="section" id="feedforward-neural-network-structure">
 <h2>2.1. Feedforward Neural Network Structure<a class="headerlink" href="#feedforward-neural-network-structure" title="Permalink to this headline">¶</a></h2>
-<img alt="https://lh6.googleusercontent.com/-h9JB8KqOcK8/UfI_lmuSTwI/AAAAAAAAGxg/DjrkXas5u5c/w926-h522-no/FeedForwardNNStruct.png" src="https://lh6.googleusercontent.com/-h9JB8KqOcK8/UfI_lmuSTwI/AAAAAAAAGxg/DjrkXas5u5c/w926-h522-no/FeedForwardNNStruct.png" />
+<img alt="https://lh4.googleusercontent.com/-zt-5emltAGw/Ufp9V81KYlI/AAAAAAAAHAE/BUtwvYS9wHU/w833-h467-no/FeedforwardNNStruct.png" src="https://lh4.googleusercontent.com/-zt-5emltAGw/Ufp9V81KYlI/AAAAAAAAHAE/BUtwvYS9wHU/w833-h467-no/FeedforwardNNStruct.png" />
 <p><img alt="addBias" src="https://lh3.googleusercontent.com/-ZoJswguVmxQ/UeZHe6Y8zeI/AAAAAAAAGpk/RDza9WEoHzk/w48-h44-no/addBias.png" style="width: 40px; height: 40px;" /> : add bias to the input. <img alt="weights" src="https://lh6.googleusercontent.com/-4RBDIb7_VRY/UeZHiRa8nrI/AAAAAAAAGps/XCWAa7E8vGA/w89-h51-no/weights.png" style="width: 60px; height: 30px;" /> : multiple the input by the corresponding weights of that layer. <img alt="save_icon" src="https://lh6.googleusercontent.com/-dmBSipdmYWY/UeZHnDw6L7I/AAAAAAAAGp0/iNiHZykQ2Lc/s14-no/save_icon.gif" style="width: 20px; height: 20px;" /> : save the result as the values of this layer. <img alt="sigmoid" src="https://lh4.googleusercontent.com/-MC8-zpYp6aU/UeZHpFFwCkI/AAAAAAAAGp8/sGCCrSxa9hI/w73-h40-no/sigmoid.png" style="width: 60px; height: 30px;" /> : execute sigmoid function on the input element-wise.</p>
 </div>
 <div class="section" id="feedforward-neural-network-implementation">
 <tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple">
 <li><strong>layersExOutputLy</strong> &#8211; A tuple that contains all layers except the output layer.</li>
 <li><strong>weightDecayParam</strong> &#8211; Parameter that regulates network model complexity.</li>
-<li><strong>sparsity</strong> &#8211; Parameter that set the threshold of the sparsity of the neural network.</li>
+<li><strong>sparsity</strong> &#8211; Parameter that sets the target value of the sparsity of the neural network.</li>
 <li><strong>sparseParam</strong> &#8211; Parameter that regulates the sparsity.</li>
 </ul>
 </td>
 <div class="section" id="details">
 <h3>2.2.1. Details<a class="headerlink" href="#details" title="Permalink to this headline">¶</a></h3>
 <p>Each layer is treated like a column vector. However, internally, layers are implemented as 1D numpy array, which is rather a row vector. So when we produce the linear combination of inputs by multiplying weights, we actually do:
-<img class="math" src="_images/math/105eb0af1e6f4ec2b098d83bf64a8ac1f344ac87.png" alt="layer*forwardWeights.T"/> rather than <img class="math" src="_images/math/82d87a088b372da97ad82d49c18802a8f6139a27.png" alt="forwardWeights*layer"/>.</p>
+<span class="math">\(layer*forwardWeights.T\)</span> rather than <span class="math">\(forwardWeights*layer\)</span>.</p>
 </div>
 </div>
 </div>
           <a href="genindex.html" title="General Index"
              >index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="costFunc.html" title="3. Cost Functions"
              >next</a> |</li>
         <li class="right" >

File deepArch/demo.html

     <script type="text/javascript" src="_static/jquery.js"></script>
     <script type="text/javascript" src="_static/underscore.js"></script>
     <script type="text/javascript" src="_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="index.html" />
     <link rel="next" title="5. Terminologies" href="jargon.html" />
     <link rel="prev" title="3. Cost Functions" href="costFunc.html" /> 
           <a href="genindex.html" title="General Index"
              accesskey="I">index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="jargon.html" title="5. Terminologies"
              accesskey="N">next</a> |</li>
         <li class="right" >
 <p>Apply my implementation of deep architectures on various data set.</p>
 <div class="section" id="feedforward-neural-network">
 <h2>4.1. Feedforward Neural Network<a class="headerlink" href="#feedforward-neural-network" title="Permalink to this headline">¶</a></h2>
-<p>I use the data set in <a class="reference external" href="http://www.kaggle.com/c/digit-recognizer">Digit Recognizer</a> competition on Kaggle, and achieved 97.371% accuracy on public test set. You can check out the demonstration in ipython notebook through ipython notebook viewer <a class="reference external" href="http://nbviewer.ipython.org/5990038">here</a>.</p>
+<p>I use the data set in <a class="reference external" href="http://www.kaggle.com/c/digit-recognizer">Digit Recognizer</a> competition on Kaggle, and achieved 97.371% accuracy on public test set. You can check out the demonstration in ipython notebook through ipython nbviewer <a class="reference external" href="http://nbviewer.ipython.org/5990038">here</a>.</p>
+</div>
+<div class="section" id="sparse-autoencoder">
+<h2>4.2. Sparse Autoencoder<a class="headerlink" href="#sparse-autoencoder" title="Permalink to this headline">¶</a></h2>
+<p>I use the IMAGE.mat data set in <a class="reference external" href="http://ufldl.stanford.edu/wiki/index.php/Exercise:Sparse_Autoencoder">sparseae_exercise.zip on UFLDL Tutorial Exercise:Sparse_Autoencoder</a>. You can check out the demonstration in ipython notebook through ipython nbviewer <a class="reference external" href="http://nbviewer.ipython.org/6132034">here</a>.</p>
 </div>
 </div>
 
   <ul>
 <li><a class="reference internal" href="#">4. Demonstrations</a><ul>
 <li><a class="reference internal" href="#feedforward-neural-network">4.1. Feedforward Neural Network</a></li>
+<li><a class="reference internal" href="#sparse-autoencoder">4.2. Sparse Autoencoder</a></li>
 </ul>
 </li>
 </ul>
           <a href="genindex.html" title="General Index"
              >index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="jargon.html" title="5. Terminologies"
              >next</a> |</li>
         <li class="right" >

File deepArch/genindex.html

     <script type="text/javascript" src="_static/jquery.js"></script>
     <script type="text/javascript" src="_static/underscore.js"></script>
     <script type="text/javascript" src="_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="index.html" /> 
   </head>
   <body>
         <li class="right" style="margin-right: 10px">
           <a href="#" title="General Index"
              accesskey="I">index</a></li>
-        <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="index.html">deepArch 0.1 documentation</a> &raquo;</li> 
       </ul>
     </div>  
 <table style="width: 100%" class="indextable genindextable"><tr>
   <td style="width: 33%" valign="top"><dl>
       
-  <dt><a href="costFunc.html#module-CostFunc">CostFunc (module)</a>
-  </dt>
-
-      
   <dt><a href="costFunc.html#CostFunc.courseraML_CostFunc">courseraML_CostFunc() (in module CostFunc)</a>
   </dt>
 
   <dt><a href="deepArchitecture.html#FeedforwardNeuNet.sigmoid">sigmoid() (in module FeedforwardNeuNet)</a>
   </dt>
 
+      
+  <dt><a href="costFunc.html#CostFunc.sparse_CostFunc">sparse_CostFunc() (in module CostFunc)</a>
+  </dt>
+
+  </dl></td>
+  <td style="width: 33%" valign="top"><dl>
+      
+  <dt><a href="costFunc.html#CostFunc.sparse_CostFuncGrad">sparse_CostFuncGrad() (in module CostFunc)</a>
+  </dt>
+
   </dl></td>
 </tr></table>
 
         <li class="right" style="margin-right: 10px">
           <a href="#" title="General Index"
              >index</a></li>
-        <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="index.html">deepArch 0.1 documentation</a> &raquo;</li> 
       </ul>
     </div>

File deepArch/index.html

     <script type="text/javascript" src="_static/jquery.js"></script>
     <script type="text/javascript" src="_static/underscore.js"></script>
     <script type="text/javascript" src="_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="#" />
     <link rel="next" title="1. Introduction" href="introduction.html" /> 
   </head>
           <a href="genindex.html" title="General Index"
              accesskey="I">index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="introduction.html" title="1. Introduction"
              accesskey="N">next</a> |</li>
         <li><a href="#">deepArch 0.1 documentation</a> &raquo;</li> 
 </ul>
 </li>
 <li class="toctree-l1"><a class="reference internal" href="costFunc.html">3. Cost Functions</a><ul>
-<li class="toctree-l2"><a class="reference internal" href="costFunc.html#module-CostFunc">3.1. Stanford Coursera Machine Learning course</a><ul>
-<li class="toctree-l3"><a class="reference internal" href="costFunc.html#references">3.1.1. References</a></li>
-</ul>
-</li>
+<li class="toctree-l2"><a class="reference internal" href="costFunc.html#stanford-coursera-machine-learning-course">3.1. Stanford Coursera Machine Learning course</a></li>
+<li class="toctree-l2"><a class="reference internal" href="costFunc.html#stanford-ufldl-tutorial">3.2. Stanford UFLDL Tutorial</a></li>
+<li class="toctree-l2"><a class="reference internal" href="costFunc.html#references">3.3. References</a></li>
 </ul>
 </li>
 <li class="toctree-l1"><a class="reference internal" href="demo.html">4. Demonstrations</a><ul>
 <li class="toctree-l2"><a class="reference internal" href="demo.html#feedforward-neural-network">4.1. Feedforward Neural Network</a></li>
+<li class="toctree-l2"><a class="reference internal" href="demo.html#sparse-autoencoder">4.2. Sparse Autoencoder</a></li>
 </ul>
 </li>
 <li class="toctree-l1"><a class="reference internal" href="jargon.html">5. Terminologies</a></li>
           <a href="genindex.html" title="General Index"
              >index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="introduction.html" title="1. Introduction"
              >next</a> |</li>
         <li><a href="#">deepArch 0.1 documentation</a> &raquo;</li> 

File deepArch/introduction.html

     <script type="text/javascript" src="_static/jquery.js"></script>
     <script type="text/javascript" src="_static/underscore.js"></script>
     <script type="text/javascript" src="_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="index.html" />
     <link rel="next" title="2. Deep Architectures" href="deepArchitecture.html" />
     <link rel="prev" title="Welcome to deepArch’s documentation!" href="index.html" /> 
           <a href="genindex.html" title="General Index"
              accesskey="I">index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="deepArchitecture.html" title="2. Deep Architectures"
              accesskey="N">next</a> |</li>
         <li class="right" >
           <a href="genindex.html" title="General Index"
              >index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="deepArchitecture.html" title="2. Deep Architectures"
              >next</a> |</li>
         <li class="right" >

File deepArch/jargon.html

     <script type="text/javascript" src="_static/jquery.js"></script>
     <script type="text/javascript" src="_static/underscore.js"></script>
     <script type="text/javascript" src="_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="index.html" />
     <link rel="prev" title="4. Demonstrations" href="demo.html" /> 
   </head>
           <a href="genindex.html" title="General Index"
              accesskey="I">index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="demo.html" title="4. Demonstrations"
              accesskey="P">previous</a> |</li>
         <li><a href="index.html">deepArch 0.1 documentation</a> &raquo;</li> 
           <a href="genindex.html" title="General Index"
              >index</a></li>
         <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
-        <li class="right" >
           <a href="demo.html" title="4. Demonstrations"
              >previous</a> |</li>
         <li><a href="index.html">deepArch 0.1 documentation</a> &raquo;</li> 

File deepArch/objects.inv

Binary file modified.

File deepArch/search.html

     <script type="text/javascript" src="_static/jquery.js"></script>
     <script type="text/javascript" src="_static/underscore.js"></script>
     <script type="text/javascript" src="_static/doctools.js"></script>
+    <script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
     <script type="text/javascript" src="_static/searchtools.js"></script>
     <link rel="top" title="deepArch 0.1 documentation" href="index.html" />
   <script type="text/javascript">
         <li class="right" style="margin-right: 10px">
           <a href="genindex.html" title="General Index"
              accesskey="I">index</a></li>
-        <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="index.html">deepArch 0.1 documentation</a> &raquo;</li> 
       </ul>
     </div>  
         <li class="right" style="margin-right: 10px">
           <a href="genindex.html" title="General Index"
              >index</a></li>
-        <li class="right" >
-          <a href="py-modindex.html" title="Python Module Index"
-             >modules</a> |</li>
         <li><a href="index.html">deepArch 0.1 documentation</a> &raquo;</li> 
       </ul>
     </div>

File deepArch/searchindex.js

-Search.setIndex({objects:{"":{CostFunc:[2,0,1,""]},"FeedforwardNeuNet.FeedforwardNeuNet":{forwardPropogateOneInput:[5,1,1,""],train:[5,1,1,""],forwardPropogateAllInput:[5,1,1,""],"__new__":[5,4,1,""]},CostFunc:{courseraML_CostFuncGrad:[2,3,1,""],courseraML_CostFunc:[2,3,1,""]},"FeedforwardNeuNet.NnLayer":{updateForwardWeight:[5,1,1,""],recvAndActvByOneInput:[5,1,1,""],actvByAllInput:[5,1,1,""],"__new__":[5,4,1,""]},FeedforwardNeuNet:{sigmoid:[5,3,1,""],NnLayer:[5,2,1,""],FeedforwardNeuNet:[5,2,1,""]}},terms:{all:[5,2],numofunitnextlv:5,partial:[5,2],execut:5,sigmoid:5,over:[5,2],layersexoutputli:5,paramtooptim:5,scalar:2,code:1,through:4,edu:2,threshold:5,follow:2,actvbyallinput:5,newweight:5,paramet:[5,2],feedforward:[0,5,4],fit:5,weightdecayparam:[5,2],activ:5,except:5,param:5,matrix:5,add:5,input:[5,1,2],hidden:2,feedforwardneunet:[5,2],modul:[0,2],match:2,"case":5,contrain:5,sourc:[5,1,2],"return":[5,2],costfunc:[5,2],variou:[4,5],ufldl:2,self2d:5,initi:5,bia:5,numofunit:5,zmnoatzigik:3,courseraml_costfunc:2,"class":[5,2],introduct:[3,0,2],sparsiti:5,layer1:5,layer0:5,like:5,level:5,edit:2,list:5,"public":4,achiev:4,vector:[5,2],either:[5,1],truth:[5,1],each:[5,2],output:[5,1,2],kaggl:4,where:2,page:0,mean:1,wiki:2,arr:5,set:[4,5],repres:[5,1,2],weight:[5,2],inputarr2d:[5,2],neunet:5,some:1,intern:5,cost:[0,5,2],sec:2,flatten:2,arg:[5,2],http:[3,2],deriv:[5,2],index:[0,2],what:3,network:[0,5,2,4],per:[5,2],label:[5,1,2],current:2,forwardpropogateallinput:5,biasunitvalu:5,row:[5,2],method:5,refer:[0,1,2],machin:[0,2],linear:5,contain:[5,2],costfuncgrad:5,here:4,multilay:2,ipython:4,modifi:5,valu:5,search:0,last:[5,2],architectur:[3,0,5,1,4],befor:2,column:5,howev:5,instanc:[5,2],implement:[0,5,2,4],whole:5,produc:5,comment:1,updateforwardweight:5,ethem:2,ground:[5,1],appli:4,arrai:[5,2],number:[5,2],terminolog:[0,1],"2nd":2,comput:[5,2],total:2,perceptron:2,unit:[5,2],given:5,from:2,describ:2,numpi:5,transform:5,next:5,nnlayer:5,treat:5,which:5,forward:[5,2],type:5,store:5,"function":[0,5,2],regul:[5,2],tupl:5,ufldl_tutori:2,coursera:[0,2],ani:5,train:[5,2],deep:[3,0,5,1,4],actual:[5,1],educ:3,than:5,must:2,serv:5,forwardpropogateoneinput:5,save:5,wise:5,target:[5,1,2],stanford:[0,2],provid:5,recvandactvbyoneinput:5,youtu:3,structur:[0,5],project:3,defin:1,activfunc:5,can:[4,5],learn:[3,0,1,2],"static":5,purpos:3,result:5,propag:5,layer:[5,1,2],constant:5,creat:3,viewer:4,certain:5,deeparch:0,sometim:1,digit:4,indic:0,lectur:2,maxit:5,itself:5,out:[3,4],want:5,tabl:0,need:5,courseraml_costfuncgrad:2,element:5,check:[3,4],cours:[0,2],multipl:[5,2],welcom:0,weightalllay:2,rather:5,self:5,when:5,detail:[0,5],accuraci:4,note:5,also:5,complex:[5,2],pure:3,test:4,instead:5,you:[4,5],document:[0,1],singl:[5,2],updat:5,competit:4,recogn:4,layers0:5,supervis:1,multipli:5,nest:5,ndarrai:5,capabl:3,notebook:4,php:2,data:[4,1],inputarr:5,demonstr:[0,4],"__new__":5,algorithm:2,neural:[0,5,2,4],backpropag:2,correspond:[5,2],alpaydin:2,exclud:5,sparseparam:5,perform:5,exampl:[5,2],combin:5,thi:[3,5,1,2],callabl:5,model:[5,2],order:2},objtypes:{"0":"py:module","1":"py:method","2":"py:class","3":"py:function","4":"py:staticmethod"},titles:["Welcome to deepArch&#8217;s documentation!","5. Terminologies","3. Cost Functions","1. Introduction","4. Demonstrations","2. Deep Architectures"],objnames:{"0":["py","module","Python module"],"1":["py","method","Python method"],"2":["py","class","Python class"],"3":["py","function","Python function"],"4":["py","staticmethod","Python static method"]},filenames:["index","jargon","costFunc","introduction","demo","deepArchitecture"]})
+Search.setIndex({objects:{"FeedforwardNeuNet.FeedforwardNeuNet":{forwardPropogateOneInput:[5,0,1,""],train:[5,0,1,""],forwardPropogateAllInput:[5,0,1,""],"__new__":[5,3,1,""]},CostFunc:{sparse_CostFuncGrad:[2,1,1,""],courseraML_CostFunc:[2,1,1,""],sparse_CostFunc:[2,1,1,""],courseraML_CostFuncGrad:[2,1,1,""]},"FeedforwardNeuNet.NnLayer":{updateForwardWeight:[5,0,1,""],recvAndActvByOneInput:[5,0,1,""],actvByAllInput:[5,0,1,""],"__new__":[5,3,1,""]},FeedforwardNeuNet:{sigmoid:[5,1,1,""],NnLayer:[5,2,1,""],FeedforwardNeuNet:[5,2,1,""]}},terms:{all:[5,2],numofunitnextlv:5,partial:[5,2],execut:5,sigmoid:5,over:[5,2],layersexoutputli:5,paramtooptim:5,scalar:2,code:1,through:4,edu:2,follow:2,actvbyallinput:5,newweight:5,paramet:[5,2],feedforward:[0,5,4],fit:5,weightdecayparam:[5,2],activ:[5,2],except:5,param:5,sparse_costfunc:2,add:5,exercis:[4,2],input:[5,1,2],hidden:2,introduct:[3,0,2],modul:[0,2],match:2,"case":5,contrain:5,sourc:[5,1,2],"return":[5,2],costfunc:[5,2],variou:[4,5],ufldl:[0,2,4],self2d:5,initi:5,bia:[5,2],numofunit:5,zmnoatzigik:3,courseraml_costfunc:2,sparseae_exercis:4,"class":[5,2],feedforwardneunet:[5,2],sparsiti:[5,2],layer1:5,layer0:5,like:5,level:5,edit:2,list:5,method:5,network:[0,5,2,4],achiev:4,vector:[5,2],either:[5,1],truth:[5,1],each:[5,2],output:[5,1,2],kaggl:4,where:2,page:0,mean:1,wiki:2,arr:5,set:[4,5,2],repres:[5,1,2],weight:[5,2],inputarr2d:[5,2],neunet:5,some:1,sparse_costfuncgrad:2,intern:5,cost:[0,5,2],sec:2,flatten:2,arg:[5,2],http:[3,2],deriv:[5,2],index:[0,2],what:3,matrix:[5,2],per:[5,2],label:[5,1,2],current:2,forwardpropogateallinput:5,biasunitvalu:5,row:[5,2],"public":4,refer:[0,1,2],machin:[0,2],spars:[0,2,4],linear:5,contain:[5,2],costfuncgrad:5,here:4,multilay:2,ipython:4,modifi:5,valu:5,search:0,sparse_autoencod:[4,2],last:5,architectur:[3,0,5,1,4],column:5,howev:5,instanc:[5,2],pdf:2,implement:[0,5,2,4],whole:5,produc:5,comment:1,updateforwardweight:5,ethem:2,ground:[5,1],appli:4,arrai:[5,2],number:[5,2],terminolog:[0,1],"2nd":2,averag:2,comput:[5,2],total:2,perceptron:2,unit:[5,2],given:5,describ:2,zip:4,numpi:5,transform:5,next:5,sparseae_read:2,treat:5,which:5,forward:[5,2],nnlayer:5,type:5,store:5,"function":[0,5,2],regul:[5,2],tupl:5,imag:4,coursera:[0,2],ani:5,train:[5,2],deep:[3,0,5,1,4],actual:[5,1],educ:3,than:5,must:2,serv:5,forwardpropogateoneinput:5,save:5,wise:5,target:[5,1,2],stanford:[0,2],provid:5,recvandactvbyoneinput:5,youtu:3,structur:[0,5],tutori:[0,2,4],project:3,defin:1,activfunc:5,can:[4,5],learn:[3,0,1,2],"static":5,purpos:3,result:5,propag:5,layer:[5,1,2],constant:5,creat:3,certain:5,deeparch:0,autoencod:[0,4],digit:4,indic:0,lectur:2,maxit:5,itself:5,out:[3,4],want:5,tabl:0,need:5,courseraml_costfuncgrad:2,exclud:[5,2],check:[3,4],cours:[0,2],multipl:[5,2],welcom:0,weightalllay:2,rather:5,self:5,when:5,detail:[0,5],accuraci:4,note:5,also:5,complex:[5,2],pure:3,test:4,instead:[5,2],you:[4,5],document:[0,1],singl:[5,2],updat:5,competit:4,sometim:1,recogn:4,mat:4,layers0:5,supervis:1,multipli:[5,2],nest:5,ndarrai:[5,2],capabl:3,notebook:4,nbviewer:4,php:2,data:[4,1],inputarr:5,demonstr:[0,4],"__new__":5,algorithm:2,neural:[0,5,2,4],backpropag:2,correspond:[5,2],alpaydin:2,element:5,sparseparam:[5,2],perform:5,exampl:[5,2],combin:5,thi:[3,5,1,2],callabl:5,model:[5,2],order:2},objtypes:{"0":"py:method","1":"py:function","2":"py:class","3":"py:staticmethod"},titles:["Welcome to deepArch&#8217;s documentation!","5. Terminologies","3. Cost Functions","1. Introduction","4. Demonstrations","2. Deep Architectures"],objnames:{"0":["py","method","Python method"],"1":["py","function","Python function"],"2":["py","class","Python class"],"3":["py","staticmethod","Python static method"]},filenames:["index","jargon","costFunc","introduction","demo","deepArchitecture"]})