Merge branch 'master' of https://github.com/aleksejs-fomins/snippets-…

…data-analysis

dynamical_systems/pde/pde_fdtd_3fields_stackoverflow.ipynb

@@ -0,0 +1,118 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Original equations\n",
+    "$$\n",
+    "\\begin{eqnarray}\n",
+    "    \\frac{\\partial H}{\\partial x}+\\beta_h (H - W) &=& 0 \\\\\n",
+    "    \\frac{\\partial C}{\\partial y} + \\beta_c (C - W) &=& 0 \\\\\n",
+    "    \\lambda_h \\frac{\\partial^2 W}{\\partial x^2} + \\lambda_c V\\frac{\\partial^2 W}{\\partial y^2}\n",
+    "    - \\frac{\\partial H}{\\partial x} - V\\frac{\\partial C}{\\partial y} &=& 0\n",
+    "\\end{eqnarray}\n",
+    "$$\n",
+    "\n",
+    "Central differences\n",
+    "$$\n",
+    "\\begin{eqnarray}\n",
+    "    H_{i+1, j} - H_{i-1, j} + 2\\Delta x\\beta_h H_{ij} &=& 2\\Delta x\\beta_h W_{ij} \\\\\n",
+    "    C_{i, j+1} - C_{i, j-1} + 2\\Delta y\\beta_c C_{ij} &=& 2\\Delta y\\beta_c W_{ij} \\\\\n",
+    "      \\lambda_h (W_{i+1, j} - 2W_{i,j} + W_{i-1, j})\n",
+    "    + \\lambda_c V \\eta^2 (W_{i, j+1} - 2W_{i,j} + W_{i, j-1})\n",
+    "    + 2(\\beta_h + V \\beta_c)\\Delta x^2 W_{ij}\n",
+    "    &=& 2\\beta_h \\Delta x^2 H_{ij} + 2V\\beta_c \\Delta x^2 C_{ij}\n",
+    "\\end{eqnarray}\n",
+    "$$\n",
+    "\n",
+    "Constraints\n",
+    "$$\\theta_w(0,y)=1, \\theta_w(x,0)=0$$\n",
+    "$$\\frac{\\partial \\theta_w(1,y)}{\\partial x}=\\frac{\\partial \\theta_w(x,1)}{\\partial y}=0$$\n",
+    "$$\\theta_h(0,y)=1, \\theta_c(x,0)=0$$\n",
+    "\n",
+    "* (@i = 1) : $W_{i-1,j} = 1$, $H_{i-1,j} = 1$\n",
+    "* (@j = 1) : $W_{i,j-1} = 0$, $H_{i,j-1} = 0$\n",
+    "* (@i = n) : $W_{i-1,j} = W_{i+1,j}$\n",
+    "* (@j = n) : $W_{i,j-1} = W_{i,j+1}$\n",
+    "\n",
+    "Problem: Need to find 2nd boundary condition for $H$ and $C$. Use trick - take second derivative\n",
+    "$$\n",
+    "\\begin{eqnarray}\n",
+    "    \\partial_x^2 H + \\beta_h (\\partial_x H - \\partial_x W) &=& 0 \\\\\n",
+    "    \\partial_y^2 C + \\beta_c (\\partial_y C - \\partial_y W) &=& 0\n",
+    "\\end{eqnarray}\n",
+    "$$\n",
+    "Discretize\n",
+    "$$\n",
+    "\\begin{eqnarray}\n",
+    "    (H_{i+1,j} - 2H_{i,j} + H_{i-1,j}) + \\frac{\\beta_h \\Delta X}{2} (H_{i+1, j} - H_{i-1, j})\n",
+    "        &=& \\frac{\\beta_h \\Delta X}{2} (W_{i+1, j} - W_{i-1, j}) \\\\\n",
+    "    (C_{i,j+1} - 2C_{i,j} + C_{i,j-1}) + \\frac{\\beta_c \\Delta Y}{2} (C_{i, j+1} - C_{i, j-1})\n",
+    "        &=& \\frac{\\beta_h \\Delta Y}{2} (W_{i, j+1} - W_{i, j-1}) \\\\\n",
+    "\\end{eqnarray}\n",
+    "$$\n",
+    "\n",
+    "At the outer edge derivatives of W are zero. Can solve for the outer edge of $H$ and $C$\n",
+    "\n",
+    "$$\n",
+    "\\begin{eqnarray}\n",
+    "    (\\frac{\\beta_h \\Delta X}{2} + 1) H_{i+1, j} &=& 2H_{i,j} - (1 - \\frac{\\beta_h \\Delta X}{2}) H_{i-1,j} \\\\\n",
+    "    (\\frac{\\beta_c \\Delta Y}{2} + 1) C_{i, j+1} &=& 2C_{i,j} - (1 - \\frac{\\beta_c \\Delta Y}{2}) C_{i,j-1}\n",
+    "\\end{eqnarray}\n",
+    "$$\n",
+    "Thus, at the outer boundaries, we have\n",
+    "$$\n",
+    "\\begin{eqnarray}\n",
+    "    (\\frac{\\beta_h^2 \\Delta X^2}{2} + \\beta_h \\Delta c + 1) H_{i,j} - H_{i-1,j}\n",
+    "        &=& \\Delta x\\beta_h (\\frac{\\beta_h \\Delta c}{2} + 1) W_{ij} \\\\\n",
+    "    (\\frac{\\beta_c^2 \\Delta Y^2}{2} + \\beta_c \\Delta y + 1) C_{i,j} - C_{i,j-1}\n",
+    "        &=& \\Delta y\\beta_c (\\frac{\\beta_c \\Delta y}{2} + 1) W_{ij} \\\\\n",
+    "\\end{eqnarray}\n",
+    "$$"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nEq = 3\n",
+    "nRow = 10\n",
+    "nCol = 10\n",
+    "nRowBig = nRow * nCol * nEq\n",
+    "bigMat = np.zeros((nRowBig, nRowBig))\n",
+    "\n",
+    "big_mapper = lambda iEq, iRow, iCol: iCol + iRow * nCol + iEq * nRow * nCol\n",
+    "\n",
+    "iRowBig = 0\n",
+    "for iRow in range(nRow):\n",
+    "    for iCol in range(nCol):\n",
+    "        \n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python (nest)",
+   "language": "python",
+   "name": "py36nest"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

machine_learning/ReinforcementLearning/labSolverTDEligibility.py

@@ -43,7 +43,7 @@ def policy_move(self, p, pold):
         moveidx = np.random.choice(4, p=prob)
         return moves[moveidx]
 
-    def solve(self, maxMoves=50000):
+    def solve(self, maxMoves=50000, updateVisited=True):
         p0, pf = self.labClass.p0, self.labClass.pf
 
         pold, p = p0, pf
@@ -54,9 +54,14 @@ def solve(self, maxMoves=50000):
             rewardThis = self.R0 * int(p == pf)
             if rewardThis > 0:
                 # Update only the cells that we have visited this run
+                if updateVisited:
+                    idxUpdate = self.eligibilityTrace > 0
+                else:
+                    idxUpdate = np.ones(len(self.value)).astype(bool)
+
                 idxVisited = self.eligibilityTrace > 0
-                self.value[idxVisited] = (1 - self.alpha) * self.value[idxVisited] \
-                                           + self.alpha * rewardThis * self.eligibilityTrace[idxVisited]
+                self.value[idxUpdate] = (1 - self.alpha) * self.value[idxUpdate] \
+                                           + self.alpha * rewardThis * self.eligibilityTrace[idxUpdate]
 
                 # Reset trace for each trial
                 self.eligibilityTrace = np.zeros(self.labShape)