Add files via upload

Author: YJango

MNIST with shape info TFrecord.ipynb

@@ -73,15 +73,21 @@
    "outputs": [],
    "source": [
     "# info of data\n",
-    "# if you want to retrieve the shape of certain feature, add the shape as extra feature, but do not name it as feature0_shape,\n",
-    "# otherwise, feature0_shape will be used to reshape feature0 by tfr.parser()\n",
-    "\n",
-    "df = pd.DataFrame({'name':['image','label','shape_image'],\n",
+    "df = pd.DataFrame({'name':['image','label','image_shape'],\n",
     "                  'type':['float32','int64','int64'],\n",
-    "                  'shape':[(28,28,1),(),(3,)],\n",
+    "                  'shape':[(784,),(),(3,)],\n",
     "                  'isbyte':[False,False,False],\n",
-    "                  \"length_type\":['fixed','fixed','fixed'],\n",
-    "                  \"default\":[np.NaN,np.NaN,np.NaN]})"
+    "                  \"length_type\":['var','fixed','fixed'],\n",
+    "                  \"default\":[np.NaN,np.NaN,np.NaN]})\n",
+    "'''\n",
+    "Note:\n",
+    "1. if the shape of particular feature for every example in your dataset varies, then you must set \"length_type\" as 'var'\n",
+    "     example: 'image' feature for the first example is 28x28; but for the second example is 64x64\n",
+    "2. 'image_shape' decides the final shape of 'image'\n",
+    "     example: the shape of 'image' is 784 when you write the tfrecord file, but the final shape after parsing is reshaped by the value of 'image_shape'\n",
+    "3. by default, '_shape' info will not be returned after parsing, \n",
+    "   if you want to keep '_shape' info, please use: tfr = TFrecorder(retrieve_shape=True)\n",
+    "'''"
    ]
   },
   {
@@ -123,9 +129,9 @@
        "      <th>0</th>\n",
        "      <td>NaN</td>\n",
        "      <td>False</td>\n",
-       "      <td>fixed</td>\n",
+       "      <td>var</td>\n",
        "      <td>image</td>\n",
-       "      <td>(28, 28, 1)</td>\n",
+       "      <td>(784,)</td>\n",
        "      <td>float32</td>\n",
        "    </tr>\n",
        "    <tr>\n",
@@ -142,7 +148,7 @@
        "      <td>NaN</td>\n",
        "      <td>False</td>\n",
        "      <td>fixed</td>\n",
-       "      <td>shape_image</td>\n",
+       "      <td>image_shape</td>\n",
        "      <td>(3,)</td>\n",
        "      <td>int64</td>\n",
        "    </tr>\n",
@@ -151,10 +157,10 @@
        "</div>"
       ],
       "text/plain": [
-       "   default  isbyte length_type         name        shape     type\n",
-       "0      NaN   False       fixed        image  (28, 28, 1)  float32\n",
-       "1      NaN   False       fixed        label           ()    int64\n",
-       "2      NaN   False       fixed  shape_image         (3,)    int64"
+       "   default  isbyte length_type         name   shape     type\n",
+       "0      NaN   False         var        image  (784,)  float32\n",
+       "1      NaN   False       fixed        label      ()    int64\n",
+       "2      NaN   False       fixed  image_shape    (3,)    int64"
       ]
      },
      "execution_count": 4,
@@ -232,7 +238,7 @@
     "for i in np.arange(dataset.num_examples):\n",
     "    features = {}\n",
     "    # write image of one example\n",
-    "    tfr.feature_writer(df.iloc[0], dataset.images[i].reshape((28,28,1)), features)\n",
+    "    tfr.feature_writer(df.iloc[0], dataset.images[i], features)\n",
     "    # write label of one example\n",
     "    tfr.feature_writer(df.iloc[1], dataset.labels[i], features)\n",
     "    # ******************\n",
@@ -283,7 +289,7 @@
     "    # 要写到tfrecord文件中的字典\n",
     "    features = {}\n",
     "    # 写一个样本的图片信息存到字典features中\n",
-    "    tfr.feature_writer(df.iloc[0], dataset.images[i].reshape((28,28,1)), features)\n",
+    "    tfr.feature_writer(df.iloc[0], dataset.images[i], features)\n",
     "    # 写一个样本的标签信息存到字典features中\n",
     "    tfr.feature_writer(df.iloc[1], dataset.labels[i], features)\n",
     "    # ******************\n",
@@ -320,7 +326,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "tfr = TFrecorder()\n",
+    "tfr = TFrecorder(retrieve_shape=True)\n",
     "def input_fn_maker(path, data_info_path, shuffle=False, batch_size = 1, epoch = 1, padding = None):\n",
     "    def input_fn():\n",
     "        filenames = tfr.get_filenames(path=path, shuffle=shuffle)\n",
@@ -345,7 +351,7 @@
     }
    ],
    "source": [
-    "padding_info = ({'image':[28,28,1],'label':[],'shape_image':[None]})\n",
+    "padding_info = ({'image':[28,28,1],'label':[],'image_shape':[None]})\n",
     "test_input_fn = input_fn_maker('mnist_tfrecord/test/',  'mnist_tfrecord/data_info.csv',\n",
     "                               padding = padding_info)\n",
     "train_input_fn = input_fn_maker('mnist_tfrecord/train/',  'mnist_tfrecord/data_info.csv', shuffle=True, batch_size = 512,\n",
@@ -370,7 +376,7 @@
     {
      "data": {
       "text/plain": [
-       "<matplotlib.image.AxesImage at 0x7ff2ec946400>"
+       "<matplotlib.image.AxesImage at 0x7f794a8a0e48>"
       ]
      },
      "execution_count": 12,
@@ -381,7 +387,7 @@
      "data": {
       "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAADQNJREFUeJzt3W+MVfWdx/HPZylNjPQBWLHEgnQb3bgaAzoaE3AzamxYbYKN1NQHGzbZMH2AZps0ZA1PypMmjemfrU9IpikpJtSWhFbRGBeDGylRGwejBYpQICzMgkAzJgUT0yDfPphDO8W5v3u5/84dv+9XQube8z1/vrnhM+ecOefcnyNCAPL5h7obAFAPwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+IKnP9HNjtrmdEOixiHAr83W057e9wvZB24dtP9nJugD0l9u9t9/2LEmHJD0gaVzSW5Iei4jfF5Zhzw/0WD/2/HdJOhwRRyPiz5J+IWllB+sD0EedhP96SSemvB+vpv0d2yO2x2yPdbAtAF3WyR/8pju0+MRhfUSMShqVOOwHBkkne/5xSQunvP+ipJOdtQOgXzoJ/1uSbrT9JduflfQNSdu70xaAXmv7sD8iLth+XNL/SJolaVNE7O9aZwB6qu1LfW1tjHN+oOf6cpMPgJmL8ANJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBShB9IivADSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iK8ANJEX4gKcIPJEX4gaTaHqJbkmwfk3RO0seSLkTEUDeaAtB7HYW/cm9E/LEL6wHQRxz2A0l1Gv6QtMP2Htsj3WgIQH90eti/LCJO2p4v6RXb70XErqkzVL8U+MUADBhHRHdWZG+QdD4ivl+YpzsbA9BQRLiV+do+7Ld9te3PXXot6SuS9rW7PgD91clh/3WSfm370np+HhEvd6UrAD3XtcP+ljbGYT/Qcz0/7AcwsxF+ICnCDyRF+IGkCD+QFOEHkurGU30prFq1qmFtzZo1xWVPnjxZrH/00UfF+pYtW4r1999/v2Ht8OHDxWWRF3t+ICnCDyRF+IGkCD+QFOEHkiL8QFKEH0iKR3pbdPTo0Ya1xYsX96+RaZw7d65hbf/+/X3sZLCMj483rD311FPFZcfGxrrdTt/wSC+AIsIPJEX4gaQIP5AU4QeSIvxAUoQfSIrn+VtUemb/tttuKy574MCBYv3mm28u1m+//fZifXh4uGHt7rvvLi574sSJYn3hwoXFeicuXLhQrJ89e7ZYX7BgQdvbPn78eLE+k6/zt4o9P5AU4QeSIvxAUoQfSIrwA0kRfiApwg8k1fR5ftubJH1V0pmIuLWaNk/SLyUtlnRM0qMR8UHTjc3g5/kH2dy5cxvWlixZUlx2z549xfqdd97ZVk+taDZewaFDh4r1ZvdPzJs3r2Ft7dq1xWU3btxYrA+ybj7P/zNJKy6b9qSknRFxo6Sd1XsAM0jT8EfELkkTl01eKWlz9XqzpIe73BeAHmv3nP+6iDglSdXP+d1rCUA/9PzeftsjkkZ6vR0AV6bdPf9p2wskqfp5ptGMETEaEUMRMdTmtgD0QLvh3y5pdfV6taTnu9MOgH5pGn7bz0p6Q9I/2R63/R+SvifpAdt/kPRA9R7ADML39mNgPfLII8X61q1bi/V9+/Y1rN17773FZScmLr/ANXPwvf0Aigg/kBThB5Ii/EBShB9IivADSXGpD7WZP7/8SMjevXs7Wn7VqlUNa9u2bSsuO5NxqQ9AEeEHkiL8QFKEH0iK8ANJEX4gKcIPJMUQ3ahNs6/Pvvbaa4v1Dz4of1v8wYMHr7inTNjzA0kRfiApwg8kRfiBpAg/kBThB5Ii/EBSPM+Pnlq2bFnD2quvvlpcdvbs2cX68PBwsb5r165i/dOK5/kBFBF+ICnCDyRF+IGkCD+QFOEHkiL8QFJNn+e3vUnSVyWdiYhbq2kbJK2RdLaabX1EvNSrJjFzPfjggw1rza7j79y5s1h/44032uoJk1rZ8/9M0opppv8oIpZU/wg+MMM0DX9E7JI00YdeAPRRJ+f8j9v+ne1Ntud2rSMAfdFu+DdK+rKkJZJOSfpBoxltj9gesz3W5rYA9EBb4Y+I0xHxcURclPQTSXcV5h2NiKGIGGq3SQDd11b4bS+Y8vZrkvZ1px0A/dLKpb5nJQ1L+rztcUnfkTRse4mkkHRM0jd72COAHuB5fnTkqquuKtZ3797dsHbLLbcUl73vvvuK9ddff71Yz4rn+QEUEX4gKcIPJEX4gaQIP5AU4QeSYohudGTdunXF+tKlSxvWXn755eKyXMrrLfb8QFKEH0iK8ANJEX4gKcIPJEX4gaQIP5AUj/Si6KGHHirWn3vuuWL9ww8/bFhbsWK6L4X+mzfffLNYx/R4pBdAEeEHkiL8QFKEH0iK8ANJEX4gKcIPJMXz/Mldc801xfrTTz9drM+aNatYf+mlxgM4cx2/Xuz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiCpps/z214o6RlJX5B0UdJoRPzY9jxJv5S0WNIxSY9GxAdN1sXz/H3W7Dp8s2vtd9xxR7F+5MiRYr30zH6zZdGebj7Pf0HStyPiZkl3S1pr+58lPSlpZ0TcKGln9R7ADNE0/BFxKiLerl6fk3RA0vWSVkraXM22WdLDvWoSQPdd0Tm/7cWSlkr6raTrIuKUNPkLQtL8bjcHoHdavrff9hxJ2yR9KyL+ZLd0WiHbI5JG2msPQK+0tOe3PVuTwd8SEb+qJp+2vaCqL5B0ZrplI2I0IoYiYqgbDQPojqbh9+Qu/qeSDkTED6eUtktaXb1eLen57rcHoFdaudS3XNJvJO3V5KU+SVqvyfP+rZIWSTou6esRMdFkXVzq67ObbrqpWH/vvfc6Wv/KlSuL9RdeeKGj9ePKtXqpr+k5f0TsltRoZfdfSVMABgd3+AFJEX4gKcIPJEX4gaQIP5AU4QeS4qu7PwVuuOGGhrUdO3Z0tO5169YV6y+++GJH60d92PMDSRF+ICnCDyRF+IGkCD+QFOEHkiL8QFJc5/8UGBlp/C1pixYt6mjdr732WrHe7PsgMLjY8wNJEX4gKcIPJEX4gaQIP5AU4QeSIvxAUlznnwGWL19erD/xxBN96gSfJuz5gaQIP5AU4QeSIvxAUoQfSIrwA0kRfiCpptf5bS+U9IykL0i6KGk0In5se4OkNZLOVrOuj4iXetVoZvfcc0+xPmfOnLbXfeTIkWL9/Pnzba8bg62Vm3wuSPp2RLxt+3OS9th+par9KCK+37v2APRK0/BHxClJp6rX52wfkHR9rxsD0FtXdM5ve7GkpZJ+W0163PbvbG+yPbfBMiO2x2yPddQpgK5qOfy250jaJulbEfEnSRslfVnSEk0eGfxguuUiYjQihiJiqAv9AuiSlsJve7Ymg78lIn4lSRFxOiI+joiLkn4i6a7etQmg25qG37Yl/VTSgYj44ZTpC6bM9jVJ+7rfHoBeaeWv/csk/Zukvbbfqaatl/SY7SWSQtIxSd/sSYfoyLvvvlus33///cX6xMREN9vBAGnlr/27JXmaEtf0gRmMO/yApAg/kBThB5Ii/EBShB9IivADSbmfQyzbZjxnoMciYrpL85/Anh9IivADSRF+ICnCDyRF+IGkCD+QFOEHkur3EN1/lPR/U95/vpo2iAa1t0HtS6K3dnWztxtanbGvN/l8YuP22KB+t9+g9jaofUn01q66euOwH0iK8ANJ1R3+0Zq3XzKovQ1qXxK9tauW3mo95wdQn7r3/ABqUkv4ba+wfdD2YdtP1tFDI7aP2d5r+526hxirhkE7Y3vflGnzbL9i+w/Vz2mHSauptw22/7/67N6x/WBNvS20/b+2D9jeb/s/q+m1fnaFvmr53Pp+2G97lqRDkh6QNC7pLUmPRcTv+9pIA7aPSRqKiNqvCdv+F0nnJT0TEbdW056SNBER36t+cc6NiP8akN42SDpf98jN1YAyC6aOLC3pYUn/rho/u0Jfj6qGz62OPf9dkg5HxNGI+LOkX0haWUMfAy8idkm6fNSMlZI2V683a/I/T9816G0gRMSpiHi7en1O0qWRpWv97Ap91aKO8F8v6cSU9+MarCG/Q9IO23tsj9TdzDSuq4ZNvzR8+vya+7lc05Gb++mykaUH5rNrZ8Trbqsj/NN9xdAgXXJYFhG3S/pXSWurw1u0pqWRm/tlmpGlB0K7I153Wx3hH5e0cMr7L0o6WUMf04qIk9XPM5J+rcEbffj0pUFSq59nau7nrwZp5ObpRpbWAHx2gzTidR3hf0vSjba/ZPuzkr4haXsNfXyC7aurP8TI9tWSvqLBG314u6TV1evVkp6vsZe/MygjNzcaWVo1f3aDNuJ1LTf5VJcy/lvSLEmbIuK7fW9iGrb/UZN7e2nyicef19mb7WclDWvyqa/Tkr4j6TlJWyUtknRc0tcjou9/eGvQ27AmD13/OnLzpXPsPve2XNJvJO2VdLGavF6T59e1fXaFvh5TDZ8bd/gBSXGHH5AU4QeSIvxAUoQfSIrwA0kRfiApwg8kRfiBpP4CIJjqosJxHysAAAAASUVORK5CYII=\n",
       "text/plain": [
-       "<matplotlib.figure.Figure at 0x7ff2eca4bdd8>"
+       "<matplotlib.figure.Figure at 0x7f794a988dd8>"
       ]
      },
      "metadata": {},
@@ -390,7 +396,7 @@
    ],
    "source": [
     "sess =tf.InteractiveSession()\n",
-    "exmaple_image, exmaple_label, exmaple_shape_image = sess.run([test_inputs['image'],test_inputs['label'],test_inputs['shape_image']])\n",
+    "exmaple_image, exmaple_label, exmaple_shape_image = sess.run([test_inputs['image'],test_inputs['label'],test_inputs['image_shape']])\n",
     "print(exmaple_image.shape, exmaple_label, exmaple_shape_image)\n",
     "plt.imshow(exmaple_image.reshape((28,28)),cmap=plt.cm.gray)"
    ]