Fix some deprecation warnings

conda/conda-reqs-pip.txt

@@ -6,3 +6,4 @@ nest_asyncio>=1.4.0
 passivetotal>=2.5.3
 sumologic-sdk>=0.1.11
 splunk-sdk>=1.6.0,!=2.0.0
+packaging>=24.0

msticpy/auth/azure_auth.py

@@ -17,14 +17,11 @@
 from ..common.provider_settings import get_provider_settings
 
 # importing only_interactive_cred for client use.
-# pylint: disable=unused-import
 from .azure_auth_core import (  # noqa: F401
     AzCredentials,
     AzureCloudConfig,
     AzureCredEnvNames,
     az_connect_core,
-    list_auth_methods,
-    only_interactive_cred,
 )
 from .cred_wrapper import CredentialWrapper
 

msticpy/common/check_version.py

@@ -4,47 +4,32 @@
 # license information.
 # --------------------------------------------------------------------------
 """Check current version against PyPI."""
-import contextlib
+from importlib.metadata import version
 
-import httpx
-from pkg_resources import parse_version
+from packaging.version import Version
+from packaging.version import parse as parse_version
 
 from .._version import VERSION
-from .utility import mp_ua_header, unit_testing
 
 __version__ = VERSION
 __author__ = "Ian Hellen"
 
 
-def check_version():
+def check_version() -> None:
     """Check the current version against latest on PyPI."""
-    installed_version = parse_version(__version__)
+    installed_version: Version = parse_version(__version__)
 
     # fetch package metadata from PyPI
-    pypi_url = "https://pypi.org/pypi/msticpy/json"
-    pkg_data = {"info": {"version": "0.0.0"}, "releases": {}}
-    with contextlib.suppress(httpx.ConnectError):
-        if not unit_testing():
-            resp = httpx.get(
-                pypi_url,
-                timeout=httpx.Timeout(2.0, connect=2.0),
-                headers=mp_ua_header(),
-            )
-            if resp.status_code == 200:
-                pkg_data = resp.json()
-
-    latest_version = pkg_data.get("info", {}).get("version", None)
-    if latest_version:
-        latest_version = parse_version(latest_version)
-    elif "releases" in pkg_data:
-        latest_version = max(parse_version(s) for s in pkg_data["releases"].keys())
+    distrib_version: str = version("msticpy")
+
+    latest_version: Version = parse_version(distrib_version)
 
     print(
         "msticpy version",
         "installed:",
         installed_version,
         "latest published:",
-        latest_version if str(latest_version) != "0.0.0" else "unknown",
+        latest_version,
     )
     if installed_version < latest_version:
         print(f"A newer version of msticpy - {latest_version} is available.")

msticpy/common/pkg_config.py

@@ -13,13 +13,14 @@
 
 """
 import contextlib
+from contextlib import AbstractContextManager
 import numbers
 import os
 from collections import UserDict
-from importlib.resources import path
+
 from importlib.util import find_spec
 from pathlib import Path
-from typing import Any, Callable, Dict, Optional, Union
+from typing import Any, Callable, Dict, Optional, Tuple, Union, List
 
 import httpx
 import yaml
@@ -294,15 +295,28 @@ def _override_config(base_config: SettingsDict, new_config: SettingsDict):
 
 def _get_default_config():
     """Return the package default config file."""
-    config_path = None
     package = "msticpy"
     try:
-        with path(package, _CONFIG_FILE) as config_path:
-            return _read_config_file(config_path) if config_path else {}
+        from importlib.resources import (  # pylint: disable=import-outside-toplevel
+            files,
+            as_file,
+        )
+
+        package_path: AbstractContextManager = as_file(
+            files(package).joinpath(_CONFIG_FILE)
+        )
+    except ImportError:
+        from importlib.resources import path  # pylint: disable=import-outside-toplevel
+
+        package_path = path(package, _CONFIG_FILE)
+
+    try:
+        with package_path as config_path:
+            return _read_config_file(config_path) if config_path.exists() else {}
     except ModuleNotFoundError as mod_err:
         # if all else fails we try to find the package default config somewhere
         # in the package tree - we use the first one we find
-        pkg_root = _get_pkg_path("msticpy")
+        pkg_root: Optional[Path] = _get_pkg_path("msticpy")
         if not pkg_root:
             raise MsticpyUserConfigError(
                 f"Unable to locate the package default {_CONFIG_FILE}",
@@ -362,15 +376,19 @@ def _create_data_providers(mp_config: Dict[str, Any]) -> Dict[str, Any]:
 
 
 def get_http_timeout(
+    *,
+    timeout: Optional[int] = None,
+    def_timeout: Optional[int] = None,
     **kwargs,
 ) -> httpx.Timeout:
     """Return timeout from settings or overridden in `kwargs`."""
-    config_timeout = get_config(
+    del kwargs
+    config_timeout: Union[int, Dict, httpx.Timeout, List, Tuple] = get_config(
         "msticpy.http_timeout", get_config("http_timeout", None)
     )
-    timeout_params = kwargs.get(
-        "timeout", kwargs.get("def_timeout", config_timeout)  # type: ignore
-    )  # type: ignore
+    timeout_params: Union[int, Dict, httpx.Timeout, List[Union[float, None]], Tuple] = (
+        timeout or def_timeout or config_timeout
+    )
     if isinstance(timeout_params, dict):
         timeout_params = {
             name: _valid_timeout(val) for name, val in timeout_params.items()
@@ -389,7 +407,9 @@ def get_http_timeout(
     return httpx.Timeout(None)
 
 
-def _valid_timeout(timeout_val) -> Union[float, None]:
+def _valid_timeout(
+    timeout_val: Optional[Union[float, numbers.Real]]
+) -> Union[float, None]:
     """Return float in valid range or None."""
     if isinstance(timeout_val, numbers.Real) and float(timeout_val) >= 0.0:
         return float(timeout_val)

msticpy/common/provider_settings.py

@@ -327,7 +327,7 @@ def _fetch_secret_setting(
     if isinstance(config_setting, str):
         return config_setting
     if not isinstance(config_setting, dict):
-        return NotImplementedError(
+        raise NotImplementedError(
             "Configuration setting format not recognized.",
             f"'{setting_path}' should be a string or dictionary",
             "with either 'EnvironmentVar' or 'KeyVault' entry.",

msticpy/common/utility/package.py

@@ -10,13 +10,15 @@
 import subprocess  # nosec
 import sys
 import warnings
+from importlib.metadata import version, PackageNotFoundError
 from pathlib import Path
 from platform import python_version
 from typing import Dict, List, Optional, Tuple, Union
 
-import pkg_resources
-from IPython import get_ipython
-from IPython.display import HTML, display
+from IPython.core.getipython import get_ipython
+from IPython.core.display import HTML
+from IPython.display import display
+from packaging.requirements import Requirement
 from tqdm.auto import tqdm
 from tqdm.notebook import tqdm as tqdm_notebook
 
@@ -126,12 +128,13 @@ def check_and_install_missing_packages(  # noqa: MC0001
             required_packages = [required_packages]
     # Check package requirements against installed set
     for req in required_packages:
-        pkg_req = pkg_resources.Requirement.parse(req)
+        pkg_req = Requirement(req)
         try:
-            found_pkg = pkg_resources.working_set.find(pkg_req)
-        except pkg_resources.VersionConflict:
-            found_pkg = None
-        if found_pkg is None:
+            pkg_version = version(pkg_req.name)
+            found_pkg: bool = True
+        except PackageNotFoundError:
+            found_pkg = False
+        if not (found_pkg and pkg_version in pkg_req.specifier):
             missing_packages.append(req)
 
     if not missing_packages:

msticpy/context/azure/azure_data.py

@@ -21,8 +21,8 @@
     AzureCloudConfig,
     az_connect,
     fallback_devicecode_creds,
-    only_interactive_cred,
 )
+from ...auth.azure_auth_core import only_interactive_cred
 from ...common.exceptions import (
     MsticpyAzureConfigError,
     MsticpyImportExtraError,

msticpy/data/drivers/azure_monitor_driver.py

@@ -24,7 +24,7 @@
 import pandas as pd
 from azure.core.exceptions import HttpResponseError
 from azure.core.pipeline.policies import UserAgentPolicy
-from pkg_resources import parse_version
+from packaging.version import parse as parse_version
 
 from ..._version import VERSION
 from ...auth.azure_auth import AzureCloudConfig, az_connect

msticpy/data/drivers/kql_driver.py

@@ -15,10 +15,11 @@
 
 import pandas as pd
 from azure.core.exceptions import ClientAuthenticationError
-from IPython import get_ipython
+from IPython.core.getipython import get_ipython
 
 from ..._version import VERSION
-from ...auth.azure_auth import AzureCloudConfig, az_connect, only_interactive_cred
+from ...auth.azure_auth import AzureCloudConfig, az_connect
+from ...auth.azure_auth_core import only_interactive_cred
 from ...common.exceptions import (
     MsticpyDataQueryError,
     MsticpyImportExtraError,

msticpy/data/drivers/resource_graph_driver.py

@@ -14,7 +14,8 @@
 from pandas.core.frame import DataFrame
 
 from ..._version import VERSION
-from ...auth.azure_auth import AzureCloudConfig, az_connect, only_interactive_cred
+from ...auth.azure_auth import AzureCloudConfig, az_connect
+from ...auth.azure_auth_core import only_interactive_cred
 from ...common.exceptions import MsticpyImportExtraError, MsticpyNotConnectedError
 from ...common.utility import export
 from .driver_base import DriverBase, QuerySource

msticpy/init/pivot.py

@@ -13,8 +13,6 @@
 from types import ModuleType
 from typing import Any, Callable, Dict, Iterable, Optional, Type
 
-import pkg_resources
-
 from .._version import VERSION
 from ..common.timespan import TimeSpan
 from ..context.tilookup import TILookup
@@ -186,7 +184,14 @@ def _get_provider_by_type(
 
     @staticmethod
     def _get_def_pivot_reg():
-        return pkg_resources.resource_filename("msticpy", _DEF_PIVOT_REG_FILE)
+        try:
+            from importlib.resources import (  # pylint: disable=import-outside-toplevel
+                files,  # noqa=attr_defined
+            )
+
+            return files("msticpy").joinpath(_DEF_PIVOT_REG_FILE)
+        except ImportError:
+            return Path(__file__).parent.parent.joinpath(_DEF_PIVOT_REG_FILE)
 
     @property
     def providers(self) -> Dict[str, Any]:

msticpy/init/pivot_core/pivot_pd_accessor.py

@@ -16,9 +16,10 @@
 
 import numpy as np
 import pandas as pd
-from IPython import get_ipython
-from IPython.display import HTML, display
-from pkg_resources import parse_version
+from IPython.core.display import HTML
+from IPython.core.getipython import get_ipython
+from IPython.display import display
+from packaging.version import parse as parse_version
 
 from ..._version import VERSION
 

msticpy/resources/mpconfig_defaults.yaml

@@ -123,6 +123,11 @@ TIProviders:
       AuthKey: *cred_key
     Primary: bool(default=False)
     Provider: "IPQualityScore"
+  AbuseIPDB:
+    Args:
+      AuthKey: *cred_key
+    Primary: bool(default=False)
+    Provider: "AbuseIPDB"
 OtherProviders:
   GeoIPLite:
     Args:

requirements-all.txt

@@ -39,6 +39,7 @@ nest_asyncio>=1.4.0
 networkx>=2.2
 numpy>=1.15.4  # pandas
 openpyxl>=3.0
+packaging>=24.0
 pandas>=1.4.0, <3.0.0
 panel>=0.14.4
 passivetotal>=2.5.3

requirements.txt

@@ -28,6 +28,7 @@ msrestazure>=0.6.0
 nest_asyncio>=1.4.0
 networkx>=2.2
 numpy>=1.15.4  # pandas
+packaging>=24.0
 pandas>=1.4.0, <3.0.0
 pydantic>=1.8.0, <3.0.0
 pygments>=2.0.0

tests/auth/test_azure_auth_core.py

@@ -16,8 +16,8 @@
     AzureCloudConfig,
     _build_env_client,
     check_cli_credentials,
-    default_auth_methods,
 )
+from msticpy.auth.cloud_mappings import default_auth_methods
 
 from ..unit_test_lib import custom_mp_config, get_test_data_path
 

tests/common/test_pkg_config.py

@@ -131,11 +131,13 @@ def test_validate_config():
         xf_id__save = os.environ.get("XFORCE_ID", "")
         xf_auth_save = os.environ.get("XFORCE_KEY", "")
         xf_auth_save = os.environ.get("MAXMIND_AUTH", "")
+        xf_auth_save = os.environ.get("IPSTACK_AUTH", "")
         # set to some value
         os.environ["VTAUTHKEY"] = "myXfId"
         os.environ["XFORCE_ID"] = "myXfId"
         os.environ["XFORCE_KEY"] = "myXfId"
         os.environ["MAXMIND_AUTH"] = "myXfId"
+        os.environ["IPSTACK_AUTH"] = "myXfId"
         pkg_config.refresh_config()
         results = pkg_config.validate_config()
         check.equal(results, ([], []))

tests/context/test_ip_utils.py

@@ -6,7 +6,6 @@
 """IP Utils test class."""
 import os
 import re
-from pathlib import Path
 from unittest.mock import patch
 
 import pandas as pd
@@ -20,7 +19,7 @@
     get_asn_from_name,
     get_ip_type,
     get_whois_df,
-    get_whois_info,
+    ip_whois,
 )
 
 from ..unit_test_lib import TEST_DATA_PATH, get_test_data_path
@@ -453,13 +452,9 @@ def test_get_whois(mock_asn_whois_query):
     respx.get(re.compile(r"http://rdap\.arin\.net/.*")).respond(200, json=RDAP_RESPONSE)
     ms_ip = "13.107.4.50"
     ms_asn = "MICROSOFT-CORP"
-    asn, _ = get_whois_info(ms_ip)
+    asn, _ = ip_whois(ms_ip)
     check.is_in(ms_asn, asn)
 
-    asn, _ = get_whois_info(IPV4["Private"][0])
-    invalid_type = "No ASN Information for IP type: Private"
-    check.equal(asn, invalid_type)
-
 
 @respx.mock
 @patch("msticpy.context.ip_utils._asn_whois_query")
@@ -486,7 +481,6 @@ def test_get_whois_df(mock_asn_whois_query, net_df):
 @respx.mock
 @pytest.fixture(scope="module")
 @patch("msticpy.context.ip_utils._asn_whois_query")
-@pytest.mark.filterwarnings("ignore::DeprecationWarning")
 def test_whois_pdext(net_df, mock_asn_whois_query):
     """Test IP Whois."""
     net_df = net_df.head(25)