Pydantic models generation

README.md

@@ -8,7 +8,8 @@
 ![Example](/etc/convert.png)
 
 json2python-models is a [Python](https://www.python.org/) tool that can generate Python models classes 
-(dataclasses, [attrs](https://github.com/python-attrs/attrs)) from JSON dataset. 
+([pydantic](https://github.com/samuelcolvin/pydantic), dataclasses, [attrs](https://github.com/python-attrs/attrs)) 
+from JSON dataset. 
 
 ## Features
 
@@ -17,11 +18,11 @@ json2python-models is a [Python](https://www.python.org/) tool that can generate
 * Fields and models **names** generation (unicode support included)
 * Similar **models generalization**
 * Handling **recursive data** structures (i.e family tree)
-* Detecting **string literals** (i.e. datetime or just stringify numbers) 
-    and providing decorators to easily convert into Python representation
+* Detecting **string serializable types** (i.e. datetime or just stringify numbers)
+* Detecting fields containing string constants (`Literal['foo', 'bar']`)
 * Generation models as **tree** (nested models) or **list**
 * Specifying when dictionaries should be processed as **`dict` type** (by default every dict is considered as some model)
-* **CLI** tool
+* **CLI** API with a lot of options
 
 ## Table of Contents
 
@@ -38,7 +39,26 @@ json2python-models is a [Python](https://www.python.org/) tool that can generate
 * [Contributing](#contributing)
 * [License](#license)
 
-## Example
+## Examples
+
+### Part of Path of Exile public items API
+
+```python
+from pydantic import BaseModel, Field
+from typing import List, Optional
+from typing_extensions import Literal
+
+
+class Tab(BaseModel):
+    id_: str = Field(..., alias="id")
+    public: bool
+    stash_type: Literal["CurrencyStash", "NormalStash", "PremiumStash"] = Field(..., alias="stashType")
+    items: List['Item']
+    account_name: Optional[str] = Field(None, alias="accountName")
+    last_character_name: Optional[str] = Field(None, alias="lastCharacterName")
+    stash: Optional[str] = None
+    league: Optional[Literal["Hardcore", "Standard"]] = None
+```
 
 ### F1 Season Results
 
@@ -83,47 +103,46 @@ driver_standings.json
 ```
 
 ```
-json2models -f attrs -l DriverStandings driver_standings.json
+json2models -f pydantic -s flat -l DriverStandings - driver_standings.json
 ```
 
 ```python
-import attr
-from json_to_models.dynamic_typing import IntString, IsoDateString
+r"""
+generated by json2python-models v0.1.2 at Mon May  4 17:46:30 2020
+command: /opt/projects/json2python-models/venv/bin/json2models -f pydantic -s flat -l DriverStandings - driver_standings.json
+"""
+from pydantic import BaseModel, Field
 from typing import List
-
-
-@attr.s
-class DriverStandings:
-    @attr.s
-    class DriverStanding:
-        @attr.s
-        class Driver:
-            driver_id: str = attr.ib()
-            permanent_number: IntString = attr.ib(converter=IntString)
-            code: str = attr.ib()
-            url: str = attr.ib()
-            given_name: str = attr.ib()
-            family_name: str = attr.ib()
-            date_of_birth: IsoDateString = attr.ib(converter=IsoDateString)
-            nationality: str = attr.ib()
-
-        @attr.s
-        class Constructor:
-            constructor_id: str = attr.ib()
-            url: str = attr.ib()
-            name: str = attr.ib()
-            nationality: str = attr.ib()
-
-        position: IntString = attr.ib(converter=IntString)
-        position_text: IntString = attr.ib(converter=IntString)
-        points: IntString = attr.ib(converter=IntString)
-        wins: IntString = attr.ib(converter=IntString)
-        driver: 'Driver' = attr.ib()
-        constructors: List['Constructor'] = attr.ib()
-
-    season: IntString = attr.ib(converter=IntString)
-    round: IntString = attr.ib(converter=IntString)
-    driver_standings: List['DriverStanding'] = attr.ib()
+from typing_extensions import Literal
+
+class DriverStandings(BaseModel):
+    season: int
+    round_: int = Field(..., alias="round")
+    DriverStandings: List['DriverStanding']
+
+class DriverStanding(BaseModel):
+    position: int
+    position_text: int = Field(..., alias="positionText")
+    points: int
+    wins: int
+    driver: 'Driver' = Field(..., alias="Driver")
+    constructors: List['Constructor'] = Field(..., alias="Constructors")
+
+class Driver(BaseModel):
+    driver_id: str = Field(..., alias="driverId")
+    permanent_number: int = Field(..., alias="permanentNumber")
+    code: str
+    url: str
+    given_name: str = Field(..., alias="givenName")
+    family_name: str = Field(..., alias="familyName")
+    date_of_birth: str = Field(..., alias="dateOfBirth")
+    nationality: str
+
+class Constructor(BaseModel):
+    constructor_id: str = Field(..., alias="constructorId")
+    url: str
+    name: str
+    nationality: Literal["Austrian", "German", "American", "British", "Italian", "French"]
 ```
 
 </p>
@@ -139,14 +158,19 @@ class DriverStandings:
 It requires a lit bit of tweaking:
 * Some fields store routes/models specs as dicts
 * There is a lot of optinal fields so we reduce merging threshold 
+* Disable string literals
 
 ```
-json_to_models -s flat -f dataclasses -m Swagger testing_tools/swagger.json 
-    --dict-keys-fields securityDefinitions paths responses definitions properties 
-    --merge percent_50 number
+json2models -s flat -f dataclasses -m Swagger testing_tools/swagger.json \
+    --dict-keys-fields securityDefinitions paths responses definitions properties \
+    --merge percent_50 number --max-strings-literals 0
 ```
 
 ```python
+r"""
+generated by json2python-models v0.1.2 at Mon May  4 18:08:09 2020
+command: /opt/projects/json2python-models/json_to_models/__main__.py -s flat -f dataclasses -m Swagger testing_tools/swagger.json --max-strings-literals 0 --dict-keys-fields securityDefinitions paths responses definitions properties --merge percent_50 number
+"""
 from dataclasses import dataclass, field
 from json_to_models.dynamic_typing import FloatString
 from typing import Any, Dict, List, Optional, Union
@@ -192,15 +216,15 @@ class Path:
 
 @dataclass
 class Property:
-    type: str
-    format: Optional[str] = None
+    type_: str
+    format_: Optional[str] = None
     xnullable: Optional[bool] = None
     items: Optional['Item_Schema'] = None
 
 
 @dataclass
 class Property_2E:
-    type: str
+    type_: str
     title: Optional[str] = None
     read_only: Optional[bool] = None
     max_length: Optional[int] = None
@@ -209,26 +233,26 @@ class Property_2E:
     enum: Optional[List[str]] = field(default_factory=list)
     maximum: Optional[int] = None
     minimum: Optional[int] = None
-    format: Optional[str] = None
+    format_: Optional[str] = None
 
 
 @dataclass
 class Item:
-    ref: Optional[str] = None
     title: Optional[str] = None
-    type: Optional[str] = None
+    type_: Optional[str] = None
+    ref: Optional[str] = None
     max_length: Optional[int] = None
     min_length: Optional[int] = None
 
 
 @dataclass
 class Parameter_SecurityDefinition:
-    name: str
-    in_: str
+    name: Optional[str] = None
+    in_: Optional[str] = None
     required: Optional[bool] = None
     schema: Optional['Item_Schema'] = None
-    type: Optional[str] = None
     description: Optional[str] = None
+    type_: Optional[str] = None
 
 
 @dataclass
@@ -253,10 +277,10 @@ class Response:
 
 @dataclass
 class Definition_Schema:
-    ref: Optional[str] = None
+    type_: str
     required: Optional[List[str]] = field(default_factory=list)
-    type: Optional[str] = None
-    properties: Optional[Dict[str, Union['Property_2E', 'Property']]] = field(default_factory=dict)
+    properties: Optional[Dict[str, Union['Property', 'Property_2E']]] = field(default_factory=dict)
+    ref: Optional[str] = None
 ```
 
 </p>
@@ -309,8 +333,8 @@ Arguments:
 
 * `-f`, `--framework` - Model framework for which python code is generated. 
     `base` (default) mean no framework so code will be generated without any decorators and additional meta-data.
-    * **Format**: `-f {base,attrs,dataclasses,custom}`
-    * **Example**: `-f attrs`
+    * **Format**: `-f {base, pydantic, attrs, dataclasses, custom}`
+    * **Example**: `-f pydantic`
     * **Default**: `-f base`
 
 * `-s`, `--structure` - Models composition style.
@@ -327,6 +351,13 @@ Arguments:
 
 * `--strings-converters` - Enable generation of string types converters (i.e. `IsoDatetimeString` or `BooleanString`).
     * **Default**: disabled
+
+* `--max-strings-literals` -  Generate `Literal['foo', 'bar']` when field have less than NUMBER string constants as values.
+    * **Format**: `--max-strings-literals <NUMBER>` 
+    * **Default**: 10 (generator classes could override it)
+    * **Example**: `--max-strings-literals 5` - only 5 literals will be saved and used to code generation
+    * **Note**: There could not be more than **15** literals per field (for performance reasons)
+    * **Note**: `attrs` code generator do not use Literals and just generate `str` fields instead
 
 * `--merge` - Merge policy settings. Possible values are: 
     * **Format**: `--merge MERGE_POLICY [MERGE_POLICY ...]`
@@ -369,7 +400,7 @@ One of model arguments (`-m` or `-l`) is required.
 
 ### Low level API
 
-> Coming soon (Wiki)
+\-
 
 ## Tests
 

json_to_models/cli.py

@@ -17,6 +17,7 @@
 from .models.attr import AttrsModelCodeGenerator
 from .models.base import GenericModelCodeGenerator, generate_code
 from .models.dataclasses import DataclassModelCodeGenerator
+from .models.pydantic import PydanticModelCodeGenerator
 from .models.structure import compose_models, compose_models_flat
 from .registry import (
     ModelCmp, ModelFieldsEquals, ModelFieldsNumberMatch, ModelFieldsPercentMatch, ModelRegistry
@@ -42,14 +43,17 @@ class Cli:
     MODEL_GENERATOR_MAPPING: Dict[str, Type[GenericModelCodeGenerator]] = {
         "base": convert_args(GenericModelCodeGenerator),
         "attrs": convert_args(AttrsModelCodeGenerator, meta=bool_js_style),
-        "dataclasses": convert_args(DataclassModelCodeGenerator, meta=bool_js_style, post_init_converters=bool_js_style)
+        "dataclasses": convert_args(DataclassModelCodeGenerator, meta=bool_js_style,
+                                    post_init_converters=bool_js_style),
+        "pydantic": convert_args(PydanticModelCodeGenerator),
     }
 
     def __init__(self):
         self.initialized = False
         self.models_data: Dict[str, Iterable[dict]] = {}  # -m/-l
         self.enable_datetime: bool = False  # --datetime
         self.strings_converters: bool = False  # --strings-converters
+        self.max_literals: int = -1  # --max-strings-literals
         self.merge_policy: List[ModelCmp] = []  # --merge
         self.structure_fn: STRUCTURE_FN_TYPE = None  # -s
         self.model_generator: Type[GenericModelCodeGenerator] = None  # -f & --code-generator
@@ -80,6 +84,7 @@ def parse_args(self, args: List[str] = None):
         self.enable_datetime = namespace.datetime
         disable_unicode_conversion = namespace.disable_unicode_conversion
         self.strings_converters = namespace.strings_converters
+        self.max_literals = namespace.max_strings_literals
         merge_policy = [m.split("_") if "_" in m else m for m in namespace.merge]
         structure = namespace.structure
         framework = namespace.framework
@@ -201,8 +206,11 @@ def set_args(
             m = importlib.import_module(module)
             self.model_generator = getattr(m, cls)
 
-        self.model_generator_kwargs = {} if not self.strings_converters else {'post_init_converters': True}
-        self.model_generator_kwargs['convert_unicode'] = not disable_unicode_conversion
+        self.model_generator_kwargs = dict(
+            post_init_converters=self.strings_converters,
+            convert_unicode=not disable_unicode_conversion,
+            max_literals=self.max_literals
+        )
         if code_generator_kwargs_raw:
             for item in code_generator_kwargs_raw:
                 if item[0] == '"':
@@ -276,6 +284,15 @@ def _create_argparser(cls) -> argparse.ArgumentParser:
             action="store_true",
             help="Enable generation of string types converters (i.e. IsoDatetimeString or BooleanString).\n\n"
         )
+        parser.add_argument(
+            "--max-strings-literals",
+            type=int,
+            default=GenericModelCodeGenerator.DEFAULT_MAX_LITERALS,
+            metavar='NUMBER',
+            help="Generate Literal['foo', 'bar'] when field have less than NUMBER string constants as values.\n"
+                 f"Pass 0 to disable. By default NUMBER={GenericModelCodeGenerator.DEFAULT_MAX_LITERALS}"
+                 f" (some generator classes could override it)\n\n"
+        )
         parser.add_argument(
             "--disable-unicode-conversion", "--no-unidecode",
             action="store_true",

json_to_models/dynamic_typing/__init__.py

@@ -1,7 +1,7 @@
 from .base import (
     BaseType, ImportPathList, MetaData, Null, Unknown, get_hash_string
 )
-from .complex import ComplexType, DDict, DList, DOptional, DTuple, DUnion, SingleType
+from .complex import ComplexType, DDict, DList, DOptional, DTuple, DUnion, SingleType, StringLiteral
 from .models_meta import AbsoluteModelRef, ModelMeta, ModelPtr
 from .string_datetime import IsoDateString, IsoDatetimeString, IsoTimeString, register_datetime_classes
 from .string_serializable import (

json_to_models/dynamic_typing/base.py

@@ -1,5 +1,5 @@
 from inspect import isclass
-from typing import Any, Generator, Iterable, List, Tuple, Union
+from typing import Any, Dict, Generator, Iterable, List, Tuple, Type, Union
 
 ImportPathList = List[Tuple[str, Union[Iterable[str], str, None]]]
 
@@ -21,14 +21,30 @@ def replace(self, t: Union['MetaData', List['MetaData']], **kwargs) -> 'BaseType
         """
         raise NotImplementedError()
 
-    def to_typing_code(self) -> Tuple[ImportPathList, str]:
+    def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \
+            -> Tuple[ImportPathList, str]:
         """
         Return typing code that represents this metadata and import path of classes that are used in this code
 
+        :param types_style: Hints for .to_typing_code() for different type wrappers
         :return: ((module_name, (class_name, ...)), code)
         """
         raise NotImplementedError()
 
+    @classmethod
+    def get_options_for_type(
+            cls,
+            t: Union['BaseType', Type['BaseType']],
+            types_style: Dict[Union['BaseType', Type['BaseType']], dict]
+    ) -> dict:
+        t_cls = t if isclass(t) else type(t)
+        mro = t_cls.__mro__
+        for base in mro:
+            options = types_style.get(base, ...)
+            if options is not Ellipsis:
+                return options
+        return {}
+
     def to_hash_string(self) -> str:
         """
         Return unique string that can be used to generate hash of type instance.
@@ -71,7 +87,8 @@ def __iter__(self) -> Iterable['MetaData']:
     def replace(self, t: 'MetaData', **kwargs) -> 'UnknownType':
         return self
 
-    def to_typing_code(self) -> Tuple[ImportPathList, str]:
+    def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \
+            -> Tuple[ImportPathList, str]:
         return ([('typing', 'Any')], 'Any')
 
     def to_hash_string(self) -> str:
@@ -90,7 +107,8 @@ def __iter__(self) -> Iterable['MetaData']:
     def replace(self, t: 'MetaData', **kwargs) -> 'NoneType':
         return self
 
-    def to_typing_code(self) -> Tuple[ImportPathList, str]:
+    def to_typing_code(self, types_style: Dict[Union['BaseType', Type['BaseType']], dict]) \
+            -> Tuple[ImportPathList, str]:
         return ([], 'None')
 
     def to_hash_string(self) -> str: