BUG: ordering in returned data on an index axis if had a selection filter

     e.g. had > 61 fields that were specified in a Term (weird but true)

Author: jreback

pandas/io/pytables.py

@@ -961,8 +961,9 @@ def infer(self, table):
 
     def convert(self, sel):
         """ set the values from this selection """
-        self.values = _maybe_convert(sel.values[self.cname], self.kind)
-
+        self.values = Index(_maybe_convert(sel.values[self.cname], self.kind))
+        self.factor = Categorical.from_array(self.values)
+   
     @property
     def attrs(self):
         return self.table._v_attrs
@@ -1481,6 +1482,22 @@ def create_axes(self, axes, obj, validate = True, min_itemsize = None):
         if validate:
             self.validate(existing_table)
 
+    def process_axes(self, obj):
+        """ process axes filters """
+
+        def reindex(obj, axis, filt, ordered):
+            axis_name = obj._get_axis_name(axis)
+            ordd = ordered & filt
+            ordd = sorted(ordered.get_indexer(ordd))
+            return obj.reindex_axis(ordered.take(ordd), axis = obj._get_axis_number(axis_name), copy = False)
+            
+        # apply the selection filters (but keep in the same order)
+        if self.selection.filter:
+            for axis, filt in self.selection.filter:
+                obj = reindex(obj, axis, filt, getattr(obj,obj._get_axis_name(axis)))
+
+        return obj
+
     def create_description(self, compression = None, complevel = None):
         """ create the description of the table from the axes & values """
 
@@ -1556,8 +1573,7 @@ def read(self, where=None):
 
         if not self.read_axes(where): return None
 
-        indicies = [ i.values for i in self.index_axes ]
-        factors  = [ Categorical.from_array(i) for i in indicies ]
+        factors  = [ a.factor for a in self.index_axes ]
         levels   = [ f.levels for f in factors ]
         N        = [ len(f.levels) for f in factors ]
         labels   = [ f.labels for f in factors ]
@@ -1597,7 +1613,8 @@ def read(self, where=None):
                        'appended')
 
             # reconstruct
-            long_index = MultiIndex.from_arrays(indicies)
+            long_index = MultiIndex.from_arrays([ i.values for i in self.index_axes ])
+
 
             for c in self.values_axes:
                 lp = DataFrame(c.data, index=long_index, columns=c.values)
@@ -1627,12 +1644,8 @@ def read(self, where=None):
         for axis,labels in self.non_index_axes:
             wp = wp.reindex_axis(labels,axis=axis,copy=False)
 
-        # apply the selection filters (but keep in the same order)
-        if self.selection.filter:
-            filter_axis_name = wp._get_axis_name(self.non_index_axes[0][0])
-            ordered  = getattr(wp,filter_axis_name)
-            new_axis = sorted(ordered & self.selection.filter)
-            wp = wp.reindex(**{ filter_axis_name : new_axis, 'copy' : False })
+        # apply the selection filters & axis orderings
+        wp = self.process_axes(wp)
 
         return wp
 
@@ -1792,7 +1805,7 @@ def read(self, where=None):
 
         if not self.read_axes(where): return None
 
-        index   = Index(self.index_axes[0].values)
+        index   = self.index_axes[0].values
         frames  = []
         for a in self.values_axes:
             columns = Index(a.values)
@@ -1815,16 +1828,8 @@ def read(self, where=None):
         for axis,labels in self.non_index_axes:
             df = df.reindex_axis(labels,axis=axis,copy=False)
 
-        # apply the selection filters (but keep in the same order)
-        filter_axis_name = df._get_axis_name(self.non_index_axes[0][0])
-
-        ordered = getattr(df,filter_axis_name)
-        if self.selection.filter:
-            ordd = ordered & self.selection.filter
-            ordd = sorted(ordered.get_indexer(ordd))
-            df      = df.reindex(**{ filter_axis_name : ordered.take(ordd), 'copy' : False })
-        else:
-            df      = df.reindex(**{ filter_axis_name : ordered , 'copy' : False })
+        # apply the selection filters & axis orderings
+        df = self.process_axes(df)
 
         return df
 
@@ -2185,11 +2190,11 @@ def eval(self):
 
                 # use a filter after reading
                 else:
-                    self.filter = set([ v[1] for v in values ])
+                    self.filter = (self.field,Index([ v[1] for v in values ]))
 
             else:
 
-                self.filter = set([ v[1] for v in values ])
+                self.filter = (self.field,Index([ v[1] for v in values ]))
 
         else:
 
@@ -2244,10 +2249,10 @@ def __init__(self, table, where=None):
             conds = [ t.condition for t in self.terms if t.condition is not None ]
             if len(conds):
                 self.condition = "(%s)" % ' & '.join(conds)
-            self.filter    = set()
+            self.filter = []
             for t in self.terms:
                 if t.filter is not None:
-                    self.filter |= t.filter
+                    self.filter.append(t.filter)
 
     def generate(self, where):
         """ where can be a : dict,list,tuple,string """

pandas/io/tests/test_pytables.py

@@ -1080,13 +1080,26 @@ def test_select(self):
         wp = tm.makePanel()
 
         # put/select ok
+        self.store.remove('wp')
         self.store.put('wp', wp, table=True)
         self.store.select('wp')
 
         # non-table ok (where = None)
+        self.store.remove('wp')
         self.store.put('wp2', wp, table=False)
         self.store.select('wp2')
 
+        # selection on the non-indexable with a large number of columns
+        wp = Panel(np.random.randn(100, 100, 100), items = [ 'Item%03d' % i for i in xrange(100) ],
+                   major_axis=date_range('1/1/2000', periods=100), minor_axis = [ 'E%03d' % i for i in xrange(100) ])
+
+        self.store.remove('wp')
+        self.store.append('wp', wp)
+        items = [ 'Item%03d' % i for i in xrange(80) ]
+        result = self.store.select('wp', Term('items', items))
+        expected = wp.reindex(items = items)
+        tm.assert_panel_equal(expected, result)
+
         # selectin non-table with a where
         #self.assertRaises(Exception, self.store.select,
         #                  'wp2', ('column', ['A', 'D']))