make-gutenberg-v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['./gutenberg/aleph.gutenberg.org/1/2/3/8/12386/12386.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/8/12383/12383-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/8/12383/12383.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/8/12385/12385.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/8/12385/12385-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/8/12380/12380-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/8/12380/12380.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/8/12381/12381.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12377/12377.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12373/12373.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12373/12373-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12370/12370.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12370/12370-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12374/12374-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12374/12374.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12372/12372-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12372/12372.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12376/12376-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12376/12376.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12375/12375-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12375/12375.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12378/12378-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/2/3/7/12378/12378.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10019/10019.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10019/10019-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10011/10011-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10011/10011.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10015/10015.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10014/10014.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10014/10014-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10017/10017-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10017/10017.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10018/10018-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10018/10018.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10013/10013.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10013/10013-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10010/10010.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10016/10016-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/1/10016/10016.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10025/10025-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10025/10025.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10021/10021.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10021/10021-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10027/10027.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10029/10029.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10029/10029-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10026/10026.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10026/10026-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10022/10022.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10023/10023.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10028/10028.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10024/10024-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/2/10024/10024.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10032/10032-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10032/10032.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10035/10035.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10035/10035-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10036/10036.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10036/10036-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10033/10033.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10033/10033-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10030/10030.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10030/10030-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10034/10034.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/3/10034/10034-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10006/10006.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10006/10006-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10004/10004.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10004/10004-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10001/10001.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10009/10009.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10003/10003.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10005/10005.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10005/10005-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10002/10002.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10002/10002-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10008/10008-8.zip',\n",
       " './gutenberg/aleph.gutenberg.org/1/0/0/0/10008/10008.zip',\n",
       " './gutenberg/aleph.gutenberg.org/etext02/comed10.zip']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zips = !find ./gutenberg/ -name '*.zip' \n",
    "zips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "import re\n",
    "import webdataset as wds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# writing gutenshards/gutenberg-000000.tar 0 0.0 GB 0\n",
      "# writing gutenshards/gutenberg-000001.tar 200 0.1 GB 200\n",
      "# writing gutenshards/gutenberg-000002.tar 200 0.1 GB 400\n",
      "# writing gutenshards/gutenberg-000003.tar 200 0.1 GB 600\n",
      "# writing gutenshards/gutenberg-000004.tar 200 0.1 GB 800\n",
      "# writing gutenshards/gutenberg-000005.tar 200 0.1 GB 1000\n",
      "# writing gutenshards/gutenberg-000006.tar 200 0.1 GB 1200\n",
      "1335\n"
     ]
    }
   ],
   "source": [
    "!rm -f gutenshards/*.tar\n",
    "zips = !find ./gutenberg/ -name '*.zip' \n",
    "count = 0\n",
    "writer = wds.ShardWriter(\"gutenshards/gutenberg-%06d.tar\", maxcount=200, maxsize=100e6)\n",
    "for zipname in zips:\n",
    "    zf = zipfile.ZipFile(zipname, 'r')\n",
    "    for txtname in zf.namelist():\n",
    "        if txtname.endswith('.txt'):\n",
    "            data = zf.read(txtname)\n",
    "            base = txtname[:-len('.txt')]\n",
    "            # print(zipname+\"/\"+txtname, len(data))\n",
    "            writer.write({\"__key__\": zipname+\"/\"+base, \"txt\": data})\n",
    "            count += 1\n",
    "print(count)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shards = !find gutenshards/. -name '*.tar'\n",
    "len(shards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def cut_text_between_markers(text, start=r\"\\*\\*\\* START.*\\n\", end=\"\\*\\*\\* END.*\\n\"):\n",
    "    start_index = re.search(start, text).end()\n",
    "    end_index = re.search(end, text).start()\n",
    "    if start_index == -1 or end_index == -1:\n",
    "        return None\n",
    "    return text[start_index:end_index].strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "ename": "UnicodeDecodeError",
     "evalue": "'utf-8' codec can't decode byte 0xea in position 12494: invalid continuation byte",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mUnicodeDecodeError\u001b[0m                        Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[76], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m ds \u001b[39m=\u001b[39m wds\u001b[39m.\u001b[39mWebDataset(\u001b[39m\"\u001b[39m\u001b[39mgutenshards/gutenberg-\u001b[39m\u001b[39m{\u001b[39m\u001b[39m000000..000006}.tar\u001b[39m\u001b[39m\"\u001b[39m)\u001b[39m.\u001b[39mdecode()\n\u001b[0;32m----> 2\u001b[0m \u001b[39mfor\u001b[39;00m sample \u001b[39min\u001b[39;00m ds:\n\u001b[1;32m      3\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m      4\u001b[0m         txt \u001b[39m=\u001b[39m sample[\u001b[39m\"\u001b[39m\u001b[39mtxt\u001b[39m\u001b[39m\"\u001b[39m]\n",
      "File \u001b[0;32m~/ocropus4extract/venv/lib/python3.10/site-packages/webdataset/pipeline.py:64\u001b[0m, in \u001b[0;36mDataPipeline.iterator\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Create an iterator through the entire dataset, using the given number of repetitions.\"\"\"\u001b[39;00m\n\u001b[1;32m     63\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrepetitions):\n\u001b[0;32m---> 64\u001b[0m     \u001b[39mfor\u001b[39;00m sample \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39miterator1():\n\u001b[1;32m     65\u001b[0m         \u001b[39myield\u001b[39;00m sample\n",
      "File \u001b[0;32m~/ocropus4extract/venv/lib/python3.10/site-packages/webdataset/filters.py:301\u001b[0m, in \u001b[0;36m_map\u001b[0;34m(data, f, handler)\u001b[0m\n\u001b[1;32m    299\u001b[0m     result \u001b[39m=\u001b[39m f(sample)\n\u001b[1;32m    300\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m exn:\n\u001b[0;32m--> 301\u001b[0m     \u001b[39mif\u001b[39;00m handler(exn):\n\u001b[1;32m    302\u001b[0m         \u001b[39mcontinue\u001b[39;00m\n\u001b[1;32m    303\u001b[0m     \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/ocropus4extract/venv/lib/python3.10/site-packages/webdataset/filters.py:80\u001b[0m, in \u001b[0;36mreraise_exception\u001b[0;34m(exn)\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mreraise_exception\u001b[39m(exn):\n\u001b[1;32m     76\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"Reraises the given exception; used as a handler.\u001b[39;00m\n\u001b[1;32m     77\u001b[0m \n\u001b[1;32m     78\u001b[0m \u001b[39m    :param exn: exception\u001b[39;00m\n\u001b[1;32m     79\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m---> 80\u001b[0m     \u001b[39mraise\u001b[39;00m exn\n",
      "File \u001b[0;32m~/ocropus4extract/venv/lib/python3.10/site-packages/webdataset/filters.py:299\u001b[0m, in \u001b[0;36m_map\u001b[0;34m(data, f, handler)\u001b[0m\n\u001b[1;32m    297\u001b[0m \u001b[39mfor\u001b[39;00m sample \u001b[39min\u001b[39;00m data:\n\u001b[1;32m    298\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 299\u001b[0m         result \u001b[39m=\u001b[39m f(sample)\n\u001b[1;32m    300\u001b[0m     \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m exn:\n\u001b[1;32m    301\u001b[0m         \u001b[39mif\u001b[39;00m handler(exn):\n",
      "File \u001b[0;32m~/ocropus4extract/venv/lib/python3.10/site-packages/webdataset/autodecode.py:558\u001b[0m, in \u001b[0;36mDecoder.__call__\u001b[0;34m(self, sample)\u001b[0m\n\u001b[1;32m    553\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Decode an entire sample.\u001b[39;00m\n\u001b[1;32m    554\u001b[0m \n\u001b[1;32m    555\u001b[0m \u001b[39m:param sample: the sample\u001b[39;00m\n\u001b[1;32m    556\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    557\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(sample, \u001b[39mdict\u001b[39m), (\u001b[39mlen\u001b[39m(sample), sample)\n\u001b[0;32m--> 558\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdecode(sample)\n",
      "File \u001b[0;32m~/ocropus4extract/venv/lib/python3.10/site-packages/webdataset/autodecode.py:549\u001b[0m, in \u001b[0;36mDecoder.decode\u001b[0;34m(self, sample)\u001b[0m\n\u001b[1;32m    547\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    548\u001b[0m         \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(v, \u001b[39mbytes\u001b[39m), \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mk,v = \u001b[39m\u001b[39m{\u001b[39;00mk\u001b[39m}\u001b[39;00m\u001b[39m, \u001b[39m\u001b[39m{\u001b[39;00mv\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[0;32m--> 549\u001b[0m         result[k] \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdecode1(k, v)\n\u001b[1;32m    550\u001b[0m \u001b[39mreturn\u001b[39;00m result\n",
      "File \u001b[0;32m~/ocropus4extract/venv/lib/python3.10/site-packages/webdataset/autodecode.py:514\u001b[0m, in \u001b[0;36mDecoder.decode1\u001b[0;34m(self, key, data)\u001b[0m\n\u001b[1;32m    512\u001b[0m key \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m+\u001b[39m key\n\u001b[1;32m    513\u001b[0m \u001b[39mfor\u001b[39;00m f \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mhandlers:\n\u001b[0;32m--> 514\u001b[0m     result \u001b[39m=\u001b[39m f(key, data)\n\u001b[1;32m    515\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(result, Continue):\n\u001b[1;32m    516\u001b[0m         key, data \u001b[39m=\u001b[39m result\u001b[39m.\u001b[39mkey, result\u001b[39m.\u001b[39mdata\n",
      "File \u001b[0;32m~/ocropus4extract/venv/lib/python3.10/site-packages/webdataset/autodecode.py:190\u001b[0m, in \u001b[0;36mbasichandlers\u001b[0;34m(key, data)\u001b[0m\n\u001b[1;32m    187\u001b[0m extension \u001b[39m=\u001b[39m re\u001b[39m.\u001b[39msub(\u001b[39mr\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m.*[.]\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m, key)\n\u001b[1;32m    189\u001b[0m \u001b[39mif\u001b[39;00m extension \u001b[39min\u001b[39;00m decoders:\n\u001b[0;32m--> 190\u001b[0m     \u001b[39mreturn\u001b[39;00m decoders[extension](data)\n\u001b[1;32m    192\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/ocropus4extract/venv/lib/python3.10/site-packages/webdataset/autodecode.py:146\u001b[0m, in \u001b[0;36m<lambda>\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m    140\u001b[0m     \u001b[39mimport\u001b[39;00m \u001b[39mcbor\u001b[39;00m\n\u001b[1;32m    142\u001b[0m     \u001b[39mreturn\u001b[39;00m cbor\u001b[39m.\u001b[39mloads(data)\n\u001b[1;32m    145\u001b[0m decoders \u001b[39m=\u001b[39m {\n\u001b[0;32m--> 146\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mtxt\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: data\u001b[39m.\u001b[39;49mdecode(\u001b[39m\"\u001b[39;49m\u001b[39mutf-8\u001b[39;49m\u001b[39m\"\u001b[39;49m),\n\u001b[1;32m    147\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mtext\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: data\u001b[39m.\u001b[39mdecode(\u001b[39m\"\u001b[39m\u001b[39mutf-8\u001b[39m\u001b[39m\"\u001b[39m),\n\u001b[1;32m    148\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mtranscript\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: data\u001b[39m.\u001b[39mdecode(\u001b[39m\"\u001b[39m\u001b[39mutf-8\u001b[39m\u001b[39m\"\u001b[39m),\n\u001b[1;32m    149\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mcls\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: \u001b[39mint\u001b[39m(data),\n\u001b[1;32m    150\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mcls2\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: \u001b[39mint\u001b[39m(data),\n\u001b[1;32m    151\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mindex\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: \u001b[39mint\u001b[39m(data),\n\u001b[1;32m    152\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39minx\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: \u001b[39mint\u001b[39m(data),\n\u001b[1;32m    153\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mid\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: \u001b[39mint\u001b[39m(data),\n\u001b[1;32m    154\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mjson\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: json\u001b[39m.\u001b[39mloads(data),\n\u001b[1;32m    155\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mjsn\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: json\u001b[39m.\u001b[39mloads(data),\n\u001b[1;32m    156\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mpyd\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: pickle\u001b[39m.\u001b[39mloads(data),\n\u001b[1;32m    157\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mpickle\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: pickle\u001b[39m.\u001b[39mloads(data),\n\u001b[1;32m    158\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mpth\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: torch_loads(data),\n\u001b[1;32m    159\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mten\u001b[39m\u001b[39m\"\u001b[39m: tenbin_loads,\n\u001b[1;32m    160\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mtb\u001b[39m\u001b[39m\"\u001b[39m: tenbin_loads,\n\u001b[1;32m    161\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mmp\u001b[39m\u001b[39m\"\u001b[39m: msgpack_loads,\n\u001b[1;32m    162\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mmsg\u001b[39m\u001b[39m\"\u001b[39m: msgpack_loads,\n\u001b[1;32m    163\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mnpy\u001b[39m\u001b[39m\"\u001b[39m: npy_loads,\n\u001b[1;32m    164\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mnpz\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39mlambda\u001b[39;00m data: np\u001b[39m.\u001b[39mload(io\u001b[39m.\u001b[39mBytesIO(data)),\n\u001b[1;32m    165\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mcbor\u001b[39m\u001b[39m\"\u001b[39m: cbor_loads,\n\u001b[1;32m    166\u001b[0m }\n\u001b[1;32m    169\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mbasichandlers\u001b[39m(key, data):\n\u001b[1;32m    170\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"Handle basic file decoding.\u001b[39;00m\n\u001b[1;32m    171\u001b[0m \n\u001b[1;32m    172\u001b[0m \u001b[39m    This function is usually part of the post= decoders.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    185\u001b[0m \u001b[39m    :param data: binary data to be decoded\u001b[39;00m\n\u001b[1;32m    186\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n",
      "\u001b[0;31mUnicodeDecodeError\u001b[0m: 'utf-8' codec can't decode byte 0xea in position 12494: invalid continuation byte"
     ]
    }
   ],
   "source": [
    "ds = wds.WebDataset(\"gutenshards/gutenberg-{000000..000006}.tar\").decode()\n",
    "for sample in ds:\n",
    "    txt = sample[\"txt\"]\n",
    "    content = cut_text_between_markers(txt)\n",
    "    if content is None:\n",
    "        print(\"skipping\", sample[\"__key__\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['__key__', '__url__', 'txt.gz'])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"*** START\" in txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}