notebooks/Minimal_Example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Minimal Example: Light-Dark 1D\n",
    "\n",
    "This is designed to be a minimal example to get POMCP running. The problaem is a one dimensional light-dark problem. The goal is to be near 0. Observations are noisy measuerements of position.\n",
    "\n",
    "```\n",
    "   -3-2-1 0 1 2 3\n",
    "...| | | | | | | | ...\n",
    "          G   S\n",
    "\n",
    "Here G is the goal. S is the starting location\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: could not import POMDPs.create_state into Main\n"
     ]
    }
   ],
   "source": [
    "using POMDPs\n",
    "import POMDPs: discount, isterminal, actions, initial_state_distribution, create_state, reward\n",
    "import POMDPs: generate_s, generate_o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "type LightDark1DState\n",
    "    status::Int64\n",
    "    y::Float64\n",
    "    LightDark1DState() = new()\n",
    "    LightDark1DState(x, y) = new(x, y)\n",
    "end\n",
    "Base.:(==)(s1::LightDark1DState, s2::LightDark1DState) = (s1.status == s2.status) && (s1.y == s2.y)\n",
    "Base.hash(s::LightDark1DState, h::UInt64=zero(UInt64)) = hash(s.status, hash(s.y, h));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "type LightDark1D <: POMDPs.POMDP{LightDark1DState,Int64,Float64}\n",
    "    discount_factor::Float64\n",
    "    correct_r::Float64\n",
    "    incorrect_r::Float64\n",
    "    step_size::Float64\n",
    "    movement_cost::Float64\n",
    "end\n",
    "LightDark1D() = LightDark1D(0.9, 10, -10, 1, 0)\n",
    "discount(p::LightDark1D) = p.discount_factor\n",
    "isterminal(::LightDark1D, s::LightDark1DState) = s.status < 0\n",
    "create_state(p::LightDark1D) = LightDark1DState(0,0);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sigma(x::Float64) = abs(x - 5)/sqrt(2) + 1e-2\n",
    "function generate_o(p::LightDark1D, s, a, sp::LightDark1DState, rng::AbstractRNG, o::Float64=0.0)\n",
    "    return sp.y + Base.randn(rng)*sigma(sp.y)\n",
    "end\n",
    "\n",
    "function generate_s(p::LightDark1D, s::LightDark1DState, a::Int64, rng::AbstractRNG,\n",
    "                      sp::LightDark1DState=create_state(p))\n",
    "    if s.status < 0                  # Terminal state\n",
    "        return copy(s)\n",
    "    end\n",
    "    if a == 0                   # Enter\n",
    "        sprime = LightDark1DState(-1, s.y)\n",
    "    else\n",
    "        sprime = LightDark1DState(s.status, s.y+a)\n",
    "    end\n",
    "    return sprime\n",
    "end\n",
    "\n",
    "function reward(p::LightDark1D, s::LightDark1DState, a::Int, sp::LightDark1DState)\n",
    "    if s.status < 0\n",
    "        return 0.0\n",
    "    end\n",
    "    if a == 0\n",
    "        if abs(s.y) < 1\n",
    "            return p.correct_r\n",
    "        else\n",
    "            return p.incorrect_r\n",
    "        end\n",
    "    else\n",
    "        return 0.0\n",
    "    end \n",
    "end;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "type LightDark1DActionSpace\n",
    "    actions::Vector{Int64}\n",
    "end\n",
    "actions(::LightDark1D) = LightDark1DActionSpace([-1, 0, 1]) # Left Stop Right\n",
    "Base.rand(rng::AbstractRNG, asp::LightDark1DActionSpace, a::Int64=1) = rand(rng, asp.actions)\n",
    "\n",
    "type LDNormalStateDist\n",
    "    mean::Float64\n",
    "    std::Float64\n",
    "end\n",
    "function Base.rand(rng::AbstractRNG, d::LDNormalStateDist, sample::LightDark1DState=LightDark1DState())\n",
    "    return LightDark1DState(0, d.mean + randn(rng)*d.std)\n",
    "end\n",
    "function initial_state_distribution(pomdp::LightDark1D)\n",
    "    return LDNormalStateDist(2, 3)\n",
    "end;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "using POMCP\n",
    "using POMDPToolbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "solver = POMCPDPWSolver()\n",
    "pomdp = LightDark1D()\n",
    "policy = solve(solver, pomdp);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "POMCP.jl: Particle Depletion! To fix this, you have three options:\n",
      "      1) use more tree_queries (will only work for very small problems)\n",
      "      2) implement a ParticleReinvigorator with reinvigorate!() and handle_unseen_observation()\n",
      "      3) implement a more advanced updater for the agent (POMCP can use any\n",
      "         belief/state distribution that supports rand())\n"
     ]
    }
   ],
   "source": [
    "try\n",
    "    simulate(RolloutSimulator(), pomdp, policy)\n",
    "catch ex\n",
    "    print(ex.msg)\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dealing with the Particle Depletion\n",
    "\n",
    "Since the problem is continuous, the probability of POMCP having simulated the observation that is actually observed is very small, so the default particle filter will fail. Instead, we will use the SIR particle filter from POMDPToolbox. For more information on other options for belief updates see [this notebook](https://github.com/JuliaPOMDP/POMCP.jl/blob/master/notebooks/Belief_and_Particle_Filter_Options.ipynb).\n",
    "\n",
    "In order for the particle filter to re-weight the particles, we need to define the observation distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "observation (generic function with 2 methods)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import POMDPs: observation, pdf\n",
    "\n",
    "immutable NormalDist # note distributions.jl does not support random number generators\n",
    "    mean::Float64\n",
    "    std::Float64\n",
    "end\n",
    "pdf(d::NormalDist, x::Float64) = exp(-(x-d.mean)^2/(2.0*d.std^2))/(d.std*sqrt(2.0*pi))\n",
    "function observation(p::LightDark1D, a::Int, sp::LightDark1DState)\n",
    "    return NormalDist(sp.y, sigma(sp.y))\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.6677181699666577"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "up = SIRParticleUpdater(pomdp, 500);\n",
    "simulate(RolloutSimulator(), pomdp, policy, up)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This doesn't work that well, so we would need to tune parameters."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Julia 0.5.1",
   "language": "julia",
   "name": "julia-0.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}