Cura/util/bigDataStorage.py

   1 import sys\r
   2 import cStringIO as StringIO\r
   3 \r
   4 class BigDataStorage(object):\r
   5         """\r
   6         The StringIO from python aborts with an out-of-memory error after 250MB.\r
   7         So the BigDataStorage stores data in multiple StringIOs to prevent this issue.\r
   8         """\r
   9         def __init__(self):\r
  10                 self._active = StringIO.StringIO()\r
  11                 self._list = [self._active]\r
  12                 self._read_index = None\r
  13 \r
  14         def write(self, data):\r
  15                 self._active.write(data)\r
  16                 if self._active.tell() > 1024 * 1024 * 50:\r
  17                         self._active = StringIO.StringIO()\r
  18                         self._list.append(self._active)\r
  19 \r
  20         def seekStart(self):\r
  21                 self._active = self._list[0]\r
  22                 self._active.seek(0)\r
  23                 self._read_index = 0\r
  24 \r
  25         def read(self, size=-1):\r
  26                 ret = self._active.read(size)\r
  27                 if ret == '':\r
  28                         if self._read_index + 1 < len(self._list):\r
  29                                 self._read_index += 1\r
  30                                 self._active = self._list[self._read_index]\r
  31                                 self._active.seek(0)\r
  32                                 ret = self._active.read(size)\r
  33                 return ret\r
  34 \r
  35         def replaceAtStart(self, dictionary):\r
  36                 data = self._list[0].getvalue()\r
  37                 block0 = data[0:2048]\r
  38                 block1 = StringIO.StringIO()\r
  39                 self._list[0] = StringIO.StringIO()\r
  40                 block1.write(data[2048:])\r
  41                 self._list.insert(1, block1)\r
  42                 for key, value in dictionary.items():\r
  43                         block0 = block0.replace(key, str(value))\r
  44                 self._list[0].write(block0)\r
  45 \r
  46         def __len__(self):\r
  47                 ret = 0\r
  48                 for data in self._list:\r
  49                         pos = data.tell()\r
  50                         data.seek(0, 2)\r
  51                         ret += data.tell()\r
  52                         data.seek(pos)\r
  53                 return ret\r
  54 \r
  55         def __iter__(self):\r
  56                 self._iter_index = 0\r
  57                 return self\r
  58 \r
  59         def next(self):\r
  60                 if self._iter_index < len(self._list):\r
  61                         ret = self._list[self._iter_index].readline()\r
  62                         if ret == '' or (ret[-1] != '\n' and ret[-1] != '\r'):\r
  63                                 self._iter_index += 1\r
  64                                 if self._iter_index < len(self._list):\r
  65                                         self._list[self._iter_index].seek(0)\r
  66                                 return ret + self.next()\r
  67                         return ret\r
  68                 raise StopIteration\r
  69 \r
  70         def tell(self):\r
  71                 pos = 0\r
  72                 for data in self._list[:self._iter_index]:\r
  73                         pos += data.tell()\r
  74                 if self._iter_index < len(self._list):\r
  75                         pos += self._list[self._iter_index].tell()\r
  76                 return pos\r
  77 \r
  78         def close(self):\r
  79                 pass\r
  80 \r
  81         def clone(self):\r
  82                 clone = BigDataStorage()\r
  83                 clone._list = []\r
  84                 for item in self._list:\r
  85                         clone._list.append(StringIO.StringIO(item.getvalue()))\r
  86                 clone._active = clone._list[-1]\r
  87                 return clone\r