Upgrade mypy (#1357)

* remove httpx version pin

* upgrade mypy

* moar

* boom

mypy.ini

@@ -34,3 +34,13 @@ ignore_missing_imports = True
 
 [mypy-starlette_prometheus.*]
 ignore_missing_imports = True
+
+[mypy-pluggy.*]
+ignore_missing_imports = True
+
+# nucliadb_sdk deprecated things
+[mypy-nucliadb_sdk.resource.*]
+disable_error_code = arg-type, call-arg
+
+[mypy-nucliadb_client.tests.*]
+disable_error_code = call-arg

nucliadb/nucliadb/common/cluster/settings.py

@@ -32,7 +32,7 @@ class Settings(BaseSettings):
     data_path: str = "./data/node"
     standalone_mode: bool = False
     standalone_node_port: int = Field(
-        10009,
+        default=10009,
         title="Standalone node port",
         description="Port to use for standalone nodes to communication with each other through",
     )
@@ -50,18 +50,18 @@ class Settings(BaseSettings):
 
     # Node limits
     max_shard_paragraphs: int = Field(
-        1_000_000,
+        default=1_000_000,
         title="Max shard paragraphs",
         description="Maximum number of paragraphs to target per shard",
     )
     max_shard_fields: int = Field(
-        500_000,
+        default=500_000,
         title="Max shard fields",
         description="Maximum number of fields to target per shard. "
         "If this is reached before max_shard_paragraphs, we will create a new shard",
     )
     max_node_replicas: int = Field(
-        650,
+        default=650,
         title="Max node replicas",
         description="Maximum number of shard replicas a single node will manage",
     )

nucliadb/nucliadb/ingest/settings.py

@@ -41,20 +41,22 @@ def _missing_(cls, value):
 
 
 class DriverSettings(BaseSettings):
-    driver: DriverConfig = Field(DriverConfig.NOT_SET, description="K/V storage driver")
+    driver: DriverConfig = Field(
+        default=DriverConfig.NOT_SET, description="K/V storage driver"
+    )
     driver_redis_url: Optional[str] = Field(
-        None, description="Redis URL. Example: redis://localhost:6379"
+        default=None, description="Redis URL. Example: redis://localhost:6379"
     )
     driver_tikv_url: Optional[List[str]] = Field(
-        None,
+        default=None,
         description="TiKV PD (Placement Dricer) URL. The URL to the cluster manager of TiKV. Example: tikv-pd.svc:2379",
     )
     driver_local_url: Optional[str] = Field(
-        None,
+        default=None,
         description="Local path to store data on file system. Example: /nucliadb/data/main",
     )
     driver_pg_url: Optional[str] = Field(
-        None,
+        default=None,
         description="PostgreSQL DSN. The connection string to the PG server. Example: postgres://nucliadb:nucliadb@postgres:5432/nucliadb. See the complete PostgreSQL documentation: https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING",  # noqa
     )
 

nucliadb/nucliadb/reader/api/models.py

@@ -57,7 +57,7 @@ class ResourceField(BaseModel):
     field_id: str
     value: ValueType
     extracted: ExtractedDataType
-    error: Optional[Error]
+    error: Optional[Error] = None
 
 
 FIELD_NAMES_TO_PB_TYPE_MAP = {v: k for k, v in FIELD_TYPES_MAP.items()}

nucliadb/nucliadb/reader/api/v1/knowledgebox.py

@@ -49,7 +49,7 @@ async def get_kbs(request: Request, prefix: str = "") -> KnowledgeBoxList:
     async with driver.transaction() as txn:
         response = KnowledgeBoxList()
         async for kbid, slug in KnowledgeBox.get_kbs(txn, prefix):
-            response.kbs.append(KnowledgeBoxObjSummary(slug=slug or None, uuid=kbid))
+            response.kbs.append(KnowledgeBoxObjSummary(slug=slug or None, uuid=kbid))  # type: ignore
         return response
 
 
@@ -71,7 +71,7 @@ async def get_kb(request: Request, kbid: str) -> KnowledgeBoxObj:
 
         return KnowledgeBoxObj(
             uuid=kbid,
-            slug=kb_config.slug,
+            slug=kb_config.slug,  # type: ignore
             config=KnowledgeBoxConfig.from_message(kb_config),
         )
 
@@ -98,6 +98,6 @@ async def get_kb_by_slug(request: Request, slug: str) -> KnowledgeBoxObj:
 
         return KnowledgeBoxObj(
             uuid=kbid,
-            slug=kb_config.slug,
+            slug=kb_config.slug,  # type: ignore
             config=KnowledgeBoxConfig.from_message(kb_config),
         )

nucliadb/nucliadb/reader/api/v1/resource.py

@@ -306,7 +306,7 @@ async def get_resource_field(
         await txn.abort()
         raise HTTPException(status_code=404, detail="Knowledge Box does not exist")
 
-    resource_field = ResourceField(field_id=field_id, field_type=field_type)
+    resource_field = ResourceField(field_id=field_id, field_type=field_type)  # type: ignore
 
     if ResourceFieldProperties.VALUE in show:
         value = await field.get_value()

nucliadb/nucliadb/search/search/merge.py

@@ -430,7 +430,7 @@ async def merge_relations_results(
     relations_responses: List[RelationSearchResponse],
     query: EntitiesSubgraphRequest,
 ) -> Relations:
-    relations = Relations(entities={}, graph=[])
+    relations = Relations(entities={})
 
     for entry_point in query.entry_points:
         relations.entities[entry_point.value] = EntitySubgraph(related_to=[])

nucliadb/nucliadb/tests/integration/test_labels.py

@@ -340,7 +340,7 @@ async def test_fieldmetadata_classification_labels(
     )
     payload = CreateResourcePayload(
         title="Foo",
-        texts={"text": TextField(body="my text")},
+        texts={"text": TextField(body="my text")},  # type: ignore
         fieldmetadata=[fieldmetadata],
     )
     resp = await nucliadb_writer.post(

nucliadb/nucliadb/tests/unit/http_clients/test_processing.py

@@ -105,7 +105,7 @@ async def test_status(self, client: processing.ProcessingHTTPClient, response):
 
     @pytest.mark.asyncio
     async def test_pull(self, client: processing.ProcessingHTTPClient, response):
-        response_data = processing.PullResponse(status="ok", data="foobar", msgid=1)
+        response_data = processing.PullResponse(status="ok", payload="foobar", msgid=1)
         response.status = 200
         response.text.return_value = response_data.json()
 

nucliadb/nucliadb/writer/resource/field.py

@@ -316,9 +316,9 @@ def parse_link_field(
 
     toprocess.linkfield[key] = models.LinkUpload(
         link=link_field.uri,
-        headers=link_field.headers,
-        cookies=link_field.cookies,
-        localstorage=link_field.localstorage,
+        headers=link_field.headers or {},
+        cookies=link_field.cookies or {},
+        localstorage=link_field.localstorage or {},
     )
 
 
@@ -390,7 +390,7 @@ async def parse_layout_field(
         )
 
     toprocess.layoutfield[key] = models.LayoutDiff(
-        format=lc.format, blocks=toprocess_blocks
+        format=lc.format, blocks=toprocess_blocks  # type: ignore
     )
 
 

nucliadb/nucliadb/writer/tests/unit/resources/test_field.py

@@ -66,7 +66,7 @@ def storage_mock():
     [
         FileField(password="mypassword", file=File(filename="myfile.pdf", payload="")),
         FileField(
-            password="mypassword", file=File(uri="http://external.foo/myfile.pdf")
+            password="mypassword", file=File(uri="http://external.foo/myfile.pdf")  # type: ignore
         ),
     ],
 )

nucliadb/requirements-test.txt

@@ -1,4 +1,4 @@
-httpx==0.23.0
+httpx>=0.23.0
 pytest-docker-fixtures>=1.3.17
 docker>=6.0.0,<7.0.0
 pytest-asyncio

nucliadb_models/nucliadb_models/common.py

@@ -89,9 +89,9 @@ class File(BaseModel):
     filename: Optional[str]
     content_type: str = "application/octet-stream"
     payload: Optional[str] = Field(description="Base64 encoded file content")
-    md5: Optional[str]
+    md5: Optional[str] = None
     # These are to be used for external files
-    uri: Optional[str]
+    uri: Optional[str] = None
     extra_headers: Dict[str, str] = {}
 
     @root_validator(pre=False)

nucliadb_models/nucliadb_models/extracted.py

@@ -33,8 +33,8 @@
 
 class ExtractedText(BaseModel):
     text: Optional[str]
-    split_text: Optional[Dict[str, str]]
-    deleted_splits: Optional[List[str]]
+    split_text: Optional[Dict[str, str]] = None
+    deleted_splits: Optional[List[str]] = None
 
     @classmethod
     def from_message(cls: Type[_T], message: resources_pb2.ExtractedText) -> _T:

nucliadb_models/nucliadb_models/metadata.py

@@ -234,7 +234,7 @@ def from_message(cls: Type[_T], message: resources_pb2.ComputedMetadata) -> _T:
                 FieldClassification(
                     field=FieldID(
                         field=fc.field.field,
-                        field_type=FIELD_TYPES_MAP[fc.field.field_type],
+                        field_type=FIELD_TYPES_MAP[fc.field.field_type],  # type: ignore
                     ),
                     classifications=[
                         Classification(label=c.label, labelset=c.labelset)

nucliadb_models/nucliadb_models/resource.py

@@ -91,7 +91,7 @@ class KnowledgeBoxConfig(BaseModel):
     enabled_filters: List[str] = []
     enabled_insights: List[str] = []
     disable_vectors: bool = False
-    similarity: Optional[VectorSimilarity]
+    similarity: Optional[VectorSimilarity] = None
 
     @validator("slug")
     def id_check(cls, v: str) -> str:
@@ -140,23 +140,23 @@ class KnowledgeBoxList(BaseModel):
 
 
 class ExtractedData(BaseModel):
-    text: Optional[ExtractedText]
-    metadata: Optional[FieldComputedMetadata]
-    large_metadata: Optional[LargeComputedMetadata]
-    vectors: Optional[VectorObject]
-    uservectors: Optional[UserVectorSet]
+    text: Optional[ExtractedText] = None
+    metadata: Optional[FieldComputedMetadata] = None
+    large_metadata: Optional[LargeComputedMetadata] = None
+    vectors: Optional[VectorObject] = None
+    uservectors: Optional[UserVectorSet] = None
 
 
 class TextFieldExtractedData(ExtractedData):
     pass
 
 
 class FileFieldExtractedData(ExtractedData):
-    file: Optional[FileExtractedData]
+    file: Optional[FileExtractedData] = None
 
 
 class LinkFieldExtractedData(ExtractedData):
-    link: Optional[LinkExtractedData]
+    link: Optional[LinkExtractedData] = None
 
 
 class LayoutFieldExtractedData(ExtractedData):
@@ -198,62 +198,62 @@ class FieldData(BaseModel):
 
 
 class TextFieldData(BaseModel):
-    value: Optional[FieldText]
-    extracted: Optional[TextFieldExtractedData]
-    error: Optional[Error]
+    value: Optional[FieldText] = None
+    extracted: Optional[TextFieldExtractedData] = None
+    error: Optional[Error] = None
 
 
 class FileFieldData(BaseModel):
-    value: Optional[FieldFile]
-    extracted: Optional[FileFieldExtractedData]
-    error: Optional[Error]
+    value: Optional[FieldFile] = None
+    extracted: Optional[FileFieldExtractedData] = None
+    error: Optional[Error] = None
 
 
 class LinkFieldData(BaseModel):
-    value: Optional[FieldLink]
-    extracted: Optional[LinkFieldExtractedData]
-    error: Optional[Error]
+    value: Optional[FieldLink] = None
+    extracted: Optional[LinkFieldExtractedData] = None
+    error: Optional[Error] = None
 
 
 class LayoutFieldData(BaseModel):
-    value: Optional[FieldLayout]
-    extracted: Optional[LayoutFieldExtractedData]
-    error: Optional[Error]
+    value: Optional[FieldLayout] = None
+    extracted: Optional[LayoutFieldExtractedData] = None
+    error: Optional[Error] = None
 
 
 class ConversationFieldData(BaseModel):
-    value: Optional[FieldConversation]
-    extracted: Optional[ConversationFieldExtractedData]
-    error: Optional[Error]
+    value: Optional[FieldConversation] = None
+    extracted: Optional[ConversationFieldExtractedData] = None
+    error: Optional[Error] = None
 
 
 class KeywordsetFieldData(BaseModel):
-    value: Optional[FieldKeywordset]
-    extracted: Optional[KeywordsetFieldExtractedData]
-    error: Optional[Error]
+    value: Optional[FieldKeywordset] = None
+    extracted: Optional[KeywordsetFieldExtractedData] = None
+    error: Optional[Error] = None
 
 
 class DatetimeFieldData(BaseModel):
-    value: Optional[FieldDatetime]
-    extracted: Optional[DatetimeFieldExtractedData]
-    error: Optional[Error]
+    value: Optional[FieldDatetime] = None
+    extracted: Optional[DatetimeFieldExtractedData] = None
+    error: Optional[Error] = None
 
 
 class GenericFieldData(BaseModel):
-    value: Optional[str]
-    extracted: Optional[TextFieldExtractedData]
-    error: Optional[Error]
+    value: Optional[str] = None
+    extracted: Optional[TextFieldExtractedData] = None
+    error: Optional[Error] = None
 
 
 class ResourceData(BaseModel):
-    texts: Optional[Dict[str, TextFieldData]]
-    files: Optional[Dict[str, FileFieldData]]
-    links: Optional[Dict[str, LinkFieldData]]
-    layouts: Optional[Dict[str, LayoutFieldData]]
-    conversations: Optional[Dict[str, ConversationFieldData]]
-    keywordsets: Optional[Dict[str, KeywordsetFieldData]]
-    datetimes: Optional[Dict[str, DatetimeFieldData]]
-    generics: Optional[Dict[str, GenericFieldData]]
+    texts: Optional[Dict[str, TextFieldData]] = None
+    files: Optional[Dict[str, FileFieldData]] = None
+    links: Optional[Dict[str, LinkFieldData]] = None
+    layouts: Optional[Dict[str, LayoutFieldData]] = None
+    conversations: Optional[Dict[str, ConversationFieldData]] = None
+    keywordsets: Optional[Dict[str, KeywordsetFieldData]] = None
+    datetimes: Optional[Dict[str, DatetimeFieldData]] = None
+    generics: Optional[Dict[str, GenericFieldData]] = None
 
 
 class QueueType(str, Enum):  # type: ignore
@@ -265,27 +265,27 @@ class Resource(BaseModel):
     id: str
 
     # This first block of attributes correspond to Basic fields
-    slug: Optional[str]
-    title: Optional[str]
-    summary: Optional[str]
-    icon: Optional[str]
-    layout: Optional[str]
-    thumbnail: Optional[str]
-    metadata: Optional[Metadata]
-    usermetadata: Optional[UserMetadata]
-    fieldmetadata: Optional[List[UserFieldMetadata]]
-    computedmetadata: Optional[ComputedMetadata]
-    created: Optional[datetime]
-    modified: Optional[datetime]
-    last_seqid: Optional[int]
-    last_account_seq: Optional[int]
-    queue: Optional[QueueType]
-
-    origin: Optional[Origin]
-    extra: Optional[Extra]
-    relations: Optional[List[Relation]]
-
-    data: Optional[ResourceData]
+    slug: Optional[str] = None
+    title: Optional[str] = None
+    summary: Optional[str] = None
+    icon: Optional[str] = None
+    layout: Optional[str] = None
+    thumbnail: Optional[str] = None
+    metadata: Optional[Metadata] = None
+    usermetadata: Optional[UserMetadata] = None
+    fieldmetadata: Optional[List[UserFieldMetadata]] = None
+    computedmetadata: Optional[ComputedMetadata] = None
+    created: Optional[datetime] = None
+    modified: Optional[datetime] = None
+    last_seqid: Optional[int] = None
+    last_account_seq: Optional[int] = None
+    queue: Optional[QueueType] = None
+
+    origin: Optional[Origin] = None
+    extra: Optional[Extra] = None
+    relations: Optional[List[Relation]] = None
+
+    data: Optional[ResourceData] = None
 
 
 class ResourcePagination(BaseModel):