Merge pull request #289 from hugovk/speedup

README.md

@@ -78,9 +78,9 @@ Get recent downloads:
 
 ```console
 $ pypistats recent pillow
-| last_day  | last_month | last_week |
-| --------: | ---------: | --------: |
-| 1,083,451 | 30,750,398 | 7,088,038 |
+| last_day | last_month | last_week |
+|----------|------------|-----------|
+|  792,408 | 34,524,504 | 8,311,284 |
 ```
 
 Help for another subcommand:
@@ -125,42 +125,42 @@ Get version downloads:
 
 ```console
 $ pypistats python_minor pillow --last-month
-| category | percent | downloads  |
-| -------- | ------: | ---------: |
-| 3.7      |  35.93% | 11,002,680 |
-| 3.6      |  33.00% | 10,107,822 |
-| 3.8      |  15.04% |  4,605,236 |
-| 3.9      |   5.03% |  1,540,571 |
-| 3.5      |   4.73% |  1,449,591 |
-| null     |   3.39% |  1,037,124 |
-| 2.7      |   2.84% |    870,677 |
-| 3.4      |   0.03% |     10,055 |
-| 3.10     |   0.01% |      2,863 |
-| 2.6      |   0.00% |         58 |
-| 3.3      |   0.00% |         44 |
-| 3.2      |   0.00% |         39 |
-| Total    |         | 30,626,760 |
-
-Date range: 2021-04-01 - 2021-04-30
+| category | percent |  downloads |
+|----------|---------|------------|
+| 3.7      |  33.33% | 11,948,221 |
+| 3.8      |  20.54% |  7,361,763 |
+| 3.6      |  17.60% |  6,307,352 |
+| null     |  12.54% |  4,496,338 |
+| 3.9      |  10.24% |  3,671,213 |
+| 3.5      |   2.71% |    971,117 |
+| 2.7      |   2.13% |    764,048 |
+| 3.10     |   0.89% |    319,636 |
+| 3.4      |   0.01% |      5,141 |
+| 3.11     |   0.00% |        585 |
+| 3.3      |   0.00% |        150 |
+| 3.2      |   0.00% |         13 |
+| Total    |         | 35,845,577 |
+
+Date range: 2021-10-01 - 2021-10-31
 ```
 
 The table is Markdown, ready for pasting in GitHub issues and PRs:
 
-| category | percent |  downloads |
-| -------- | ------: | ---------: |
-| 3.7      |  35.93% | 11,002,680 |
-| 3.6      |  33.00% | 10,107,822 |
-| 3.8      |  15.04% |  4,605,236 |
-| 3.9      |   5.03% |  1,540,571 |
-| 3.5      |   4.73% |  1,449,591 |
-| null     |   3.39% |  1,037,124 |
-| 2.7      |   2.84% |    870,677 |
-| 3.4      |   0.03% |     10,055 |
-| 3.10     |   0.01% |      2,863 |
-| 2.6      |   0.00% |         58 |
-| 3.3      |   0.00% |         44 |
-| 3.2      |   0.00% |         39 |
-| Total    |         | 30,626,760 |
+| category | percent | downloads  |
+| -------- | ------- | ---------- |
+| 3.7      | 33.33%  | 11,948,221 |
+| 3.8      | 20.54%  | 7,361,763  |
+| 3.6      | 17.60%  | 6,307,352  |
+| null     | 12.54%  | 4,496,338  |
+| 3.9      | 10.24%  | 3,671,213  |
+| 3.5      | 2.71%   | 971,117    |
+| 2.7      | 2.13%   | 764,048    |
+| 3.10     | 0.89%   | 319,636    |
+| 3.4      | 0.01%   | 5,141      |
+| 3.11     | 0.00%   | 585        |
+| 3.3      | 0.00%   | 150        |
+| 3.2      | 0.00%   | 13         |
+| Total    |         | 35,845,577 |
 
 These are equivalent (in May 2019):
 

setup.cfg

@@ -35,9 +35,11 @@ packages = find:
 install_requires =
     httpx>=0.19
     platformdirs
+    prettytable>=2.4.0
     pytablewriter[html]>=0.63
     python-dateutil
     python-slugify
+    importlib-metadata;python_version < '3.8'
 python_requires = >=3.6
 package_dir = =src
 setup_requires =

src/pypistats/__init__.py

@@ -10,22 +10,17 @@
 import warnings
 from pathlib import Path
 
-import httpx
-import pkg_resources
 from platformdirs import user_cache_dir
-from pytablewriter import (
-    HtmlTableWriter,
-    MarkdownTableWriter,
-    NumpyTableWriter,
-    PandasDataFrameWriter,
-    RstSimpleTableWriter,
-    String,
-    TsvTableWriter,
-)
-from pytablewriter.style import Align, Style, ThousandSeparator
 from slugify import slugify
 
-__version__ = pkg_resources.get_distribution(__name__).version
+try:
+    # Python 3.8+
+    import importlib.metadata as importlib_metadata
+except ImportError:
+    # <Python 3.7 and lower
+    import importlib_metadata
+
+__version__ = importlib_metadata.version(__name__)
 
 BASE_URL = "https://pypistats.org/api/"
 CACHE_DIR = Path(user_cache_dir("pypistats"))
@@ -116,6 +111,8 @@ def pypi_stats_api(
 
     if res == {}:
         # No cache, or couldn't load cache
+        import httpx
+
         r = httpx.get(url, headers={"User-Agent": USER_AGENT})
 
         # Raise if we made a bad request
@@ -326,12 +323,62 @@ def _percent(data):
     return data
 
 
-def _tabulate(data, format="markdown"):
+def _tabulate(data, format: str = "markdown"):
     """Return data in specified format"""
 
+    if isinstance(data, dict):
+        headers = list(data.keys())
+    else:  # isinstance(data, list):
+        headers = sorted(set().union(*(d.keys() for d in data)))
+
+    # Move downloads last
+    headers.append("downloads")
+    headers.remove("downloads")
+
+    if format == "markdown":
+        return _prettytable(headers, data)
+    else:
+        return _pytablewriter(headers, data, format)
+
+
+def _prettytable(headers, data) -> str:
+    from prettytable import MARKDOWN, PrettyTable
+
+    x = PrettyTable()
+    x.set_style(MARKDOWN)
+
+    if isinstance(data, dict):
+        data = [data]
+    for header in headers:
+        col_data = [row[header] if header in row else "" for row in data]
+        x.add_column(header, col_data)
+        x.align["last_day"] = "r"
+        x.align["last_month"] = "r"
+        x.align["last_week"] = "r"
+        x.align["category"] = "l"
+        x.align["percent"] = "r"
+        x.align["downloads"] = "r"
+        x.custom_format["last_day"] = lambda f, v: f"{v:,}"
+        x.custom_format["last_month"] = lambda f, v: f"{v:,}"
+        x.custom_format["last_week"] = lambda f, v: f"{v:,}"
+        x.custom_format["downloads"] = lambda f, v: f"{v:,}"
+
+    return x.get_string() + "\n"
+
+
+def _pytablewriter(headers, data, format: str):
+    from pytablewriter import (
+        HtmlTableWriter,
+        NumpyTableWriter,
+        PandasDataFrameWriter,
+        RstSimpleTableWriter,
+        String,
+        TsvTableWriter,
+    )
+    from pytablewriter.style import Align, Style, ThousandSeparator
+
     format_writers = {
         "html": HtmlTableWriter,
-        "markdown": MarkdownTableWriter,
         "numpy": NumpyTableWriter,
         "pandas": PandasDataFrameWriter,
         "rst": RstSimpleTableWriter,
@@ -343,15 +390,10 @@ def _tabulate(data, format="markdown"):
         writer.margin = 1
 
     if isinstance(data, dict):
-        headers = list(data.keys())
         writer.value_matrix = [data]
     else:  # isinstance(data, list):
-        headers = sorted(set().union(*(d.keys() for d in data)))
         writer.value_matrix = data
 
-    # Move downloads last
-    headers.append("downloads")
-    headers.remove("downloads")
     writer.headers = headers
 
     # Custom alignment and format

tests/data/expected_tabulated.py

@@ -64,7 +64,7 @@
 
 EXPECTED_TABULATED_MD = """
 | category |    date    | downloads |
-| -------- | ---------- | --------: |
+|:---------|:----------:|----------:|
 | 2.6      | 2018-08-15 |        51 |
 | 2.7      | 2018-08-15 |    63,749 |
 | 3.2      | 2018-08-15 |         2 |

tests/test_pypistats.py

@@ -250,10 +250,10 @@ def test__tabulate_noarg(self):
     @pytest.mark.parametrize(
         "test_input, expected",
         [
-            ("html", EXPECTED_TABULATED_HTML),
-            ("markdown", EXPECTED_TABULATED_MD),
-            ("rst", EXPECTED_TABULATED_RST),
-            ("tsv", EXPECTED_TABULATED_TSV),
+            pytest.param("html", EXPECTED_TABULATED_HTML, id="html"),
+            pytest.param("markdown", EXPECTED_TABULATED_MD, id="markdown"),
+            pytest.param("rst", EXPECTED_TABULATED_RST, id="rst"),
+            pytest.param("tsv", EXPECTED_TABULATED_TSV, id="tsv"),
         ],
     )
     def test__tabulate(self, test_input, expected):
@@ -460,7 +460,29 @@ def test_valid_json(self):
         assert json.loads(output) == json.loads(mocked_response)
 
     @respx.mock
-    def test_recent_tabular(self):
+    @pytest.mark.parametrize(
+        "test_format, expected_output",
+        [
+            pytest.param(
+                "markdown",
+                """
+|  last_day | last_month |  last_week |
+|----------:|-----------:|-----------:|
+| 2,295,765 | 67,759,913 | 15,706,750 |
+""",
+                id="markdown",
+            ),
+            pytest.param(
+                "tsv",
+                """
+"last_day"\t"last_month"\t"last_week"
+2,295,765\t67,759,913\t15,706,750
+""",
+                id="tsv",
+            ),
+        ],
+    )
+    def test_recent_tabular(self, test_format, expected_output):
         # Arrange
         package = "pip"
         mocked_url = "https://pypistats.org/api/packages/pip/recent"
@@ -469,15 +491,10 @@ def test_recent_tabular(self):
                 {"last_day": 2295765, "last_month": 67759913, "last_week": 15706750},
             "package": "pip", "type": "recent_downloads"
         }"""
-        expected_output = """
-| last_day  | last_month | last_week  |
-| --------: | ---------: | ---------: |
-| 2,295,765 | 67,759,913 | 15,706,750 |
-"""
 
         # Act
         respx.get(mocked_url).respond(content=mocked_response)
-        output = pypistats.recent(package)
+        output = pypistats.recent(package, format=test_format)
 
         # Assert
         assert output.strip() == expected_output.strip()
@@ -489,8 +506,8 @@ def test_overall_tabular_start_date(self):
         mocked_url = "https://pypistats.org/api/packages/pip/overall?&mirrors=false"
         mocked_response = SAMPLE_RESPONSE_OVERALL
         expected_output = """
-|    category     | percent | downloads |
-| --------------- | ------: | --------: |
+| category        | percent | downloads |
+|:----------------|--------:|----------:|
 | with_mirrors    | 100.00% | 1,487,218 |
 | without_mirrors |  99.24% | 1,475,979 |
 | Total           |         | 1,487,218 |
@@ -512,8 +529,8 @@ def test_overall_tabular_end_date(self):
         mocked_url = "https://pypistats.org/api/packages/pip/overall?&mirrors=false"
         mocked_response = SAMPLE_RESPONSE_OVERALL
         expected_output = """
-|    category     | percent | downloads |
-| --------------- | ------: | --------: |
+| category        | percent | downloads |
+|:----------------|--------:|----------:|
 | with_mirrors    | 100.00% | 2,100,139 |
 | without_mirrors |  99.21% | 2,083,472 |
 | Total           |         | 2,100,139 |
@@ -621,8 +638,8 @@ def test_system_tabular(self):
             "type": "system_downloads"
         }"""
         expected_output = """
-| category | percent |  downloads  |
-| -------- | ------: | ----------: |
+| category | percent |   downloads |
+|:---------|--------:|------------:|
 | Linux    |  83.14% | 236,502,274 |
 | null     |  10.75% |  30,579,325 |
 | Darwin   |   3.77% |  10,734,594 |
@@ -681,7 +698,7 @@ def test_versions_are_strings(self):
         data = copy.deepcopy(SAMPLE_DATA_VERSION_STRINGS)
         expected_output = """
 | category |    date    | downloads |
-| -------- | ---------- | --------: |
+|:---------|:----------:|----------:|
 | 3.1      | 2018-08-15 |        10 |
 | 3.10     | 2018-08-15 |         1 |
 """

tests/test_pypistats_cache.py

@@ -96,8 +96,8 @@ def test_subcommand_with_cache(self):
           "type": "overall_downloads"
         }"""
         expected_output = """
-|    category     | downloads |
-| --------------- | --------: |
+| category        | downloads |
+|:----------------|----------:|
 | without_mirrors | 2,295,765 |
 
 Date range: 2018-11-01 - 2018-11-01