Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modest performance, address #12647 #12656

Closed
wants to merge 19 commits into from
Closed

Modest performance, address #12647 #12656

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
a7df841
Modest performance, address #12647
kshedden Mar 17, 2016
7d91d51
Add test data set from raderaj
kshedden Mar 19, 2016
ea2339f
Use encoding when reading column headers
kshedden Mar 19, 2016
bdc9a06
More cython for performance, refactored constants
kshedden Mar 19, 2016
3bd1b35
Move more code to cython
kshedden Mar 19, 2016
23bdf7a
Decouple data decoding and decoding e.g. of column names
kshedden Mar 19, 2016
dc330c5
Add two missing alignment constants
kshedden Mar 19, 2016
7e156b7
Working on cython issues
kshedden Apr 10, 2016
3ef626e
Working on cython issues
kshedden Apr 10, 2016
11c2f31
Further cythonization
kshedden Apr 10, 2016
873a877
Added option to not decode header text
kshedden Apr 10, 2016
c26d22b
added to whatsnew
kshedden Apr 11, 2016
1af73b3
Further encoding work
kshedden Apr 11, 2016
8b4b96d
pep8 cleanup
kshedden Apr 11, 2016
ea87a7f
Fix encoding handling bug for py2
kshedden Apr 21, 2016
b7de358
flake8 fixes
kshedden Apr 21, 2016
fe4731b
Integrate jreback's cython improvements
kshedden Apr 22, 2016
af085f7
Add one more type
kshedden Apr 22, 2016
b3024ed
Add missing test data files
kshedden Apr 22, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Further encoding work
  • Loading branch information
@kshedden
kshedden committed Apr 21, 2016
commit 1af73b3a50288aaf97bc41b86b58a48fc0885443
30 changes: 17 additions & 13 deletions pandas/io/sas/sas7bdat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ def __init__(self, path_or_buf, index=None, convert_dates=True,
self.convert_text = convert_text
self.convert_header_text = convert_header_text

self.default_encoding = "latin-1"
self.compression = ""
self.column_names_strings = []
self.column_names = []
Expand Down Expand Up @@ -149,12 +150,12 @@ def _get_properties(self):
buf = self._read_bytes(const.dataset_offset, const.dataset_length)
self.name = buf.rstrip(b'\x00 ')
if self.convert_header_text:
self.name = self.name.decode(self.encoding)
self.name = self.name.decode(self.encoding or self.default_encoding)

buf = self._read_bytes(const.file_type_offset, const.file_type_length)
self.file_type = buf.rstrip(b'\x00 ')
if self.convert_header_text:
self.file_type = self.file_type.decode(self.encoding)
self.file_type = self.file_type.decode(self.encoding or self.default_encoding)

# Timestamp is epoch 01/01/1960
epoch = pd.datetime(1960, 1, 1)
Expand Down Expand Up @@ -183,31 +184,31 @@ def _get_properties(self):
const.sas_release_length)
self.sas_release = buf.rstrip(b'\x00 ')
if self.convert_header_text:
self.sas_release = self.sas_release.decode(self.encoding)
self.sas_release = self.sas_release.decode(self.encoding or self.default_encoding)

buf = self._read_bytes(const.sas_server_type_offset + total_align,
const.sas_server_type_length)
self.server_type = buf.rstrip(b'\x00 ')
if self.convert_header_text:
self.server_type = self.server_type.decode(self.encoding)
self.server_type = self.server_type.decode(self.encoding or self.default_encoding)

buf = self._read_bytes(const.os_version_number_offset + total_align,
const.os_version_number_length)
self.os_version = buf.rstrip(b'\x00 ')
if self.convert_header_text:
self.os_version = self.os_version.decode(self.encoding)
self.os_version = self.os_version.decode(self.encoding or self.default_encoding)

buf = self._read_bytes(const.os_name_offset + total_align,
const.os_name_length)
buf = buf.rstrip(b'\x00 ')
if len(buf) > 0:
self.os_name = buf.decode(self.encoding)
self.os_name = buf.decode(self.encoding or self.default_encoding)
else:
buf = self._read_bytes(const.os_maker_offset + total_align,
const.os_maker_length)
self.os_name = buf.rstrip(b'\x00 ')
if self.convert_header_text:
self.os_name = self.os_name.decode(self.encoding)
self.os_name = self.os_name.decode(self.encoding or self.default_encoding)

# Read a single float of the given width (4 or 8).
def _read_float(self, offset, width):
Expand Down Expand Up @@ -401,14 +402,14 @@ def _process_columntext_subheader(self, offset, length):
buf = self._read_bytes(offset, text_block_size)
cname = buf[0:text_block_size].rstrip(b"\x00 ")
if self.convert_header_text:
cname = cname.decode(self.encoding)
cname = cname.decode(self.encoding or self.default_encoding)
self.column_names_strings.append(cname)

if len(self.column_names_strings) == 1:
column_name = self.column_names_strings[0]
compression_literal = ""
for cl in const.compression_literals:
if cl in column_name:
if cl in str(column_name):
compression_literal = cl
self.compression = compression_literal
offset -= self._int_length
Expand All @@ -425,20 +426,23 @@ def _process_columntext_subheader(self, offset, length):
if self.U64:
offset1 += 4
buf = self._read_bytes(offset1, self._lcp)
self.creator_proc = buf[0:self._lcp].decode()
self.creator_proc = buf[0:self._lcp]
elif compression_literal == const.rle_compression:
offset1 = offset + 40
if self.U64:
offset1 += 4
buf = self._read_bytes(offset1, self._lcp)
self.creator_proc = buf[0:self._lcp].decode()
self.creator_proc = buf[0:self._lcp]
elif self._lcs > 0:
self._lcp = 0
offset1 = offset + 16
if self.U64:
offset1 += 4
buf = self._read_bytes(offset1, self._lcs)
self.creator_proc = buf[0:self._lcp].decode()
self.creator_proc = buf[0:self._lcp]
if self.convert_header_text:
if hasattr(self, "creator_proc"):
self.creator_proc = self.creator_proc.decode(self.encoding or self.default_encoding)

def _process_columnname_subheader(self, offset, length):
int_len = self._int_length
Expand Down Expand Up @@ -624,7 +628,7 @@ def _chunk_to_dataframe(self):
elif self.column_types[j] == b's':
rslt[name] = self._string_chunk[js, :]
if self.convert_text and (self.encoding is not None):
rslt[name] = rslt[name].str.decode(self.encoding)
rslt[name] = rslt[name].str.decode(self.encoding or self.default_encoding)
if self.blank_missing:
ii = rslt[name].str.len() == 0
rslt.loc[ii, name] = np.nan
Expand Down
19 changes: 19 additions & 0 deletions pandas/io/tests/sas/test_sas7bdat.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,25 @@ def test_from_iterator(self):
tm.assert_frame_equal(df, df0.iloc[2:5, :])


def test_encoding_options():
dirpath = tm.get_data_path()
fname = os.path.join(dirpath, "test1.sas7bdat")
df1 = pd.read_sas(fname)
df2 = pd.read_sas(fname, encoding='utf-8')
for col in df1.columns:
try:
df1[col] = df1[col].str.decode('utf-8')
except AttributeError:
pass
tm.assert_frame_equal(df1, df2)

from pandas.io.sas.sas7bdat import SAS7BDATReader
rdr = SAS7BDATReader(fname, convert_header_text=False)
df3 = rdr.read()
for x,y in zip(df1.columns, df3.columns):
assert(x == y.decode())


def test_productsales():
dirpath = tm.get_data_path()
fname = os.path.join(dirpath, "productsales.sas7bdat")
Expand Down