{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "plotgrid (generic function with 1 method)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using Plots; qwt()\n",
    "default(size=(500,300), leg=false)\n",
    "\n",
    "# creates x/y vectors which can define a grid in a zig-zag pattern\n",
    "function gridxy(lim, n::Int)\n",
    "    xs = linspace(lim..., n)\n",
    "    xypairs = vec([(x,y) for x in vcat(xs,reverse(xs)), y in xs])\n",
    "    Plots.unzip(xypairs)\n",
    "end\n",
    "\n",
    "# plot a grid from x/y vectors\n",
    "function plotgrid(x, y)\n",
    "    plot([x y], [y x], c=:black)\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The problem... can we classify the functions?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "f2 (generic function with 1 method)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# these are the functions we want to classify\n",
    "f1(x) = 0.6sin(10x) + 0.1\n",
    "f2(x) = f1(x) - 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build a neural net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO: Recompiling stale cache file /home/tom/.julia/lib/v0.4/OnlineStats.ji for module OnlineStats.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.366577 seconds (2.08 M allocations: 156.080 MB, 3.22% gc time)\n",
      "\n",
      "\n",
      "\n",
      "maxabs(β - coef(o)) for\n",
      "\n",
      "glm:      0.005542500269197448\n",
      "sgd:      NaN\n",
      "proxgrad: 0.004859362717011262\n",
      "rda:      0.007882387662070361\n"
     ]
    }
   ],
   "source": [
    "using OnlineAI\n",
    "\n",
    "# first create a neural net to separate the functions\n",
    "numInputs = 2\n",
    "numOutputs = 1\n",
    "hiddenLayerStructure = [3,3,2]\n",
    "net = buildClassificationNet(numInputs, numOutputs, hiddenLayerStructure;\n",
    "    hiddenActivation = TanhActivation(),\n",
    "    finalActivation = TanhActivation(),\n",
    "\n",
    ")\n",
    "\n",
    "# show the network\n",
    "viz = visualize(net);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# On to the fun..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pick the plotting limits\n",
    "lim = (-1,1)\n",
    "default(xlim = lim, ylim = lim)\n",
    "\n",
    "# show the grid\n",
    "n = 40\n",
    "gridx, gridy = gridxy(lim, n)\n",
    "p = plotgrid(gridx, gridy)\n",
    "\n",
    "# show the funcs\n",
    "funcs = [f1, f2]\n",
    "plot!(funcs, lim..., w=3)\n",
    "\n",
    "# kick off an animation... we can save frames whenever we want, lets save the start\n",
    "anim = Animation()\n",
    "frame(anim)\n",
    "\n",
    "# open a gui window\n",
    "gui()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# That looks tricky..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# function to sample from x's\n",
    "xsample() = rand(Distributions.Uniform(lim...)) \n",
    "\n",
    "# pick one of the functions at random, sample from the x line, then update the\n",
    "# neural net with [x, f(x)] as the inputs\n",
    "function sampleAndUpdate()\n",
    "    f = sample(funcs)\n",
    "    x = xsample()\n",
    "    y = float(f == f1)\n",
    "    update!(net, Float64[x, f(x)], [y])\n",
    "end\n",
    "\n",
    "# take x matrix and convert to the first layer's activation\n",
    "function activateHidden(net, x)\n",
    "#     input = x\n",
    "    @assert net.layers[end].nin == 2\n",
    "    proj = Array(nrows(x), 2)\n",
    "    for i in 1:nrows(x)\n",
    "        data = row(x,i)\n",
    "        for layer in net.layers[1:end-1]\n",
    "        #proj = Array(nrows(x), layer.nout)\n",
    "            OnlineAI.forward!(layer, row(proj,i), false)\n",
    "            data = layer.a\n",
    "#             row!(proj, i, layer.a)\n",
    "        end\n",
    "        row!(proj, i, data)\n",
    "#         input = proj\n",
    "    end\n",
    "    vec(proj[:,1]), vec(proj[:,2])\n",
    "end    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# update net with new samples\n",
    "for i in 1:10000\n",
    "    sampleAndUpdate()\n",
    "end\n",
    "\n",
    "# update the plot... project each series to the first hidden layer and reset the data\n",
    "x = linspace(lim..., 100)\n",
    "p[1] = activateHidden(net, hcat(gridx, gridy))\n",
    "p[2] = activateHidden(net, hcat(gridy, gridx))\n",
    "p[3] = activateHidden(net, hcat(x, map(f1,x)))\n",
    "p[4] = activateHidden(net, hcat(x, map(f2,x)))\n",
    "\n",
    "# show/update the plot\n",
    "gui(p)\n",
    "frame(anim);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# build an animated gif\n",
    "gif(anim, fps = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# update the net representation with weights, etc\n",
    "update!(viz)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "p1 = plot(rand(20))\n",
    "p2 = plot(rand(10))\n",
    "p3 = scatter(rand(100))\n",
    "p4 = plot(rand(1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "subplot(p1,p2,p3,p4, nr=1, leg=false)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "immerse()\n",
    "p = plot(rand(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "gui()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "append!(p,1,rand(10))\n",
    "gui()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.4.0",
   "language": "julia",
   "name": "julia-0.4"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}