-
-
Notifications
You must be signed in to change notification settings - Fork 18.7k
ENH: Allow compression in NDFrame.to_csv to be a dict with optional arguments (#26023) #26024
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
4e73dc4
ab7620d
2e782f9
83e8834
d238878
b41be54
60ea58c
8ba9082
0a3a9fd
a1cb3f7
af2a96c
5853a28
789751f
5b09e6f
68a2b4d
c856f50
8df6c81
40d0252
18a735d
103c877
b6c34bc
969d387
abfbc0f
04ae25d
9c22652
56a75c2
bbfea34
7717f16
779511e
780eb04
6c4e679
1b567c9
9324b63
7cf65ee
29374f3
6701aa4
0f5489d
e04138e
6f2bf00
865aa81
8d1deee
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
…atsnew
- Loading branch information
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,7 +6,7 @@ | |
import operator | ||
import pickle | ||
from textwrap import dedent | ||
from typing import Dict, FrozenSet, List, Set, Union | ||
from typing import Any, Dict, FrozenSet, List, Optional, Set, Union | ||
import warnings | ||
import weakref | ||
|
||
|
@@ -2912,7 +2912,7 @@ def to_latex(self, buf=None, columns=None, col_space=None, header=True, | |
def to_csv(self, path_or_buf=None, sep=",", na_rep='', float_format=None, | ||
columns=None, header=True, index=True, index_label=None, | ||
mode='w', encoding=None, | ||
compression: Union[str, Dict, None] = 'infer', | ||
compression: Optional[Union[str, Dict[str, Any]]] = 'infer', | ||
quoting=None, quotechar='"', line_terminator=None, | ||
chunksize=None, tupleize_cols=None, date_format=None, | ||
doublequote=True, escapechar=None, decimal='.'): | ||
|
@@ -3028,6 +3028,10 @@ def to_csv(self, path_or_buf=None, sep=",", na_rep='', float_format=None, | |
... 'weapon': ['sai', 'bo staff']}) | ||
>>> df.to_csv(index=False) | ||
'name,mask,weapon\nRaphael,red,sai\nDonatello,purple,bo staff\n' | ||
>>> compression_opts = dict(method='zip', archive_name='out.csv') | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can you put the comment before the example (and put a blank line between cases); also might need to have a DOCTEST: SKIP here There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can you do this |
||
>>> df.to_csv('out.zip', index=False, compression=compression_opts) | ||
|
||
# creates 'out.zip' containing 'out.csv' | ||
""" | ||
|
||
df = self if isinstance(self, ABCDataFrame) else self.to_frame() | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -9,7 +9,7 @@ | |
import lzma | ||
import mmap | ||
import os | ||
from typing import Any, Dict, Tuple, Union | ||
from typing import Any, Dict, Optional, Tuple, Union | ||
from urllib.error import URLError # noqa | ||
from urllib.parse import ( # noqa | ||
urlencode, urljoin, urlparse as parse_url, uses_netloc, uses_params, | ||
|
@@ -233,7 +233,7 @@ def file_path_to_url(path): | |
} | ||
|
||
|
||
def _get_compression_method(compression: Union[str, Dict, None]): | ||
def _get_compression_method(compression: Optional[Union[str, Dict[str, Any]]]): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think the Dict can be typed as There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. can you do this |
||
""" | ||
Simplifies a compression argument to a compression method string and | ||
a dict containing additional arguments. | ||
|
@@ -253,7 +253,6 @@ def _get_compression_method(compression: Union[str, Dict, None]): | |
------ | ||
ValueError on dict missing 'method' key | ||
""" | ||
compression_args = {} # type: Dict[str, Any] | ||
# Handle dict | ||
if isinstance(compression, dict): | ||
compression_args = compression.copy() | ||
|
@@ -263,6 +262,8 @@ def _get_compression_method(compression: Union[str, Dict, None]): | |
except KeyError: | ||
raise ValueError("If dict, compression " | ||
"must have key 'method'") | ||
else: | ||
compression_args = {} | ||
return compression, compression_args | ||
WillAyd marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
|
||
|
@@ -318,7 +319,7 @@ def _infer_compression(filepath_or_buffer, compression): | |
|
||
|
||
def _get_handle(path_or_buf, mode, encoding=None, | ||
compression: Union[str, Dict, None] = None, | ||
compression: Optional[Union[str, Dict[str, Any]]] = None, | ||
memory_map=False, is_text=True): | ||
""" | ||
Get file handle for given path/buffer and mode. | ||
|
@@ -464,7 +465,7 @@ class BytesZipFile(zipfile.ZipFile, BytesIO): # type: ignore | |
""" | ||
# GH 17778 | ||
def __init__(self, file, mode, compression=zipfile.ZIP_DEFLATED, | ||
archive_name: Union[str, zipfile.ZipInfo, None] = None, | ||
archive_name: Optional[str] = None, | ||
**kwargs): | ||
if mode in ['wb', 'rb']: | ||
mode = mode.replace('b', '') | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -548,6 +548,13 @@ def test_to_csv_compression_dict(self, compression_only): | |
read_df = pd.read_csv(path, index_col=0) | ||
tm.assert_frame_equal(read_df, df) | ||
|
||
def test_to_csv_compression_dict_no_method(self): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you append |
||
# GH 26023 | ||
df = DataFrame({"ABC": [1]}) | ||
compression = {"some_option": True} | ||
with tm.ensure_clean("out.zip") as path, pytest.raises(ValueError): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you match on the expected message with There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. do the raises as an inner context manager,e .g.
|
||
df.to_csv(path, compression=compression) | ||
|
||
@pytest.mark.parametrize("compression", ["zip", "infer"]) | ||
@pytest.mark.parametrize("archive_name", [None, "test_to_csv.csv", | ||
"test_to_csv.zip"]) | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
what is a ByteZipFile (meaning a user has NO idea what this is); pls rephrase.