feat: UTC-416: Implement 'Ensure all annotators are evaluated' setting for GT tasks

reference.md

@@ -8961,6 +8961,14 @@ client.projects.create()
 <dl>
 <dd>
 
+**annotator_evaluation_enabled:** `typing.Optional[bool]` — Enable annotator evaluation for the project
+
+</dd>
+</dl>
+
+<dl>
+<dd>
+
 **color:** `typing.Optional[str]` 
 
 </dd>
@@ -9583,6 +9591,14 @@ client.projects.update(
 <dl>
 <dd>
 
+**annotator_evaluation_enabled:** `typing.Optional[bool]` — Enable annotator evaluation for the project
+
+</dd>
+</dl>
+
+<dl>
+<dd>
+
 **annotator_evaluation_minimum_score:** `typing.Optional[str]` 
 
 </dd>
@@ -9599,6 +9615,14 @@ client.projects.update(
 <dl>
 <dd>
 
+**annotator_evaluation_onboarding_tasks:** `typing.Optional[int]` 
+
+</dd>
+</dl>
+
+<dl>
+<dd>
+
 **assignment_settings:** `typing.Optional[AssignmentSettingsRequest]` 
 
 </dd>

src/label_studio_sdk/projects/client.py

@@ -191,6 +191,7 @@ def list(
     def create(
         self,
         *,
+        annotator_evaluation_enabled: typing.Optional[bool] = OMIT,
         color: typing.Optional[str] = OMIT,
         control_weights: typing.Optional[typing.Optional[typing.Any]] = OMIT,
         created_by: typing.Optional[UserSimpleRequest] = OMIT,
@@ -227,6 +228,9 @@ def create(
 
         Parameters
         ----------
+        annotator_evaluation_enabled : typing.Optional[bool]
+            Enable annotator evaluation for the project
+
         color : typing.Optional[str]
 
         control_weights : typing.Optional[typing.Optional[typing.Any]]
@@ -328,6 +332,7 @@ def create(
             "api/projects/",
             method="POST",
             json={
+                "annotator_evaluation_enabled": annotator_evaluation_enabled,
                 "color": color,
                 "control_weights": control_weights,
                 "created_by": convert_and_respect_annotation_metadata(
@@ -583,8 +588,10 @@ def update(
         agreement_threshold: typing.Optional[str] = OMIT,
         annotation_limit_count: typing.Optional[int] = OMIT,
         annotation_limit_percent: typing.Optional[str] = OMIT,
+        annotator_evaluation_enabled: typing.Optional[bool] = OMIT,
         annotator_evaluation_minimum_score: typing.Optional[str] = OMIT,
         annotator_evaluation_minimum_tasks: typing.Optional[int] = OMIT,
+        annotator_evaluation_onboarding_tasks: typing.Optional[int] = OMIT,
         assignment_settings: typing.Optional[AssignmentSettingsRequest] = OMIT,
         color: typing.Optional[str] = OMIT,
         comment_classification_config: typing.Optional[str] = OMIT,
@@ -642,10 +649,15 @@ def update(
 
         annotation_limit_percent : typing.Optional[str]
 
+        annotator_evaluation_enabled : typing.Optional[bool]
+            Enable annotator evaluation for the project
+
         annotator_evaluation_minimum_score : typing.Optional[str]
 
         annotator_evaluation_minimum_tasks : typing.Optional[int]
 
+        annotator_evaluation_onboarding_tasks : typing.Optional[int]
+
         assignment_settings : typing.Optional[AssignmentSettingsRequest]
 
         color : typing.Optional[str]
@@ -775,8 +787,10 @@ def update(
                 "agreement_threshold": agreement_threshold,
                 "annotation_limit_count": annotation_limit_count,
                 "annotation_limit_percent": annotation_limit_percent,
+                "annotator_evaluation_enabled": annotator_evaluation_enabled,
                 "annotator_evaluation_minimum_score": annotator_evaluation_minimum_score,
                 "annotator_evaluation_minimum_tasks": annotator_evaluation_minimum_tasks,
+                "annotator_evaluation_onboarding_tasks": annotator_evaluation_onboarding_tasks,
                 "assignment_settings": convert_and_respect_annotation_metadata(
                     object_=assignment_settings, annotation=AssignmentSettingsRequest, direction="write"
                 ),
@@ -1398,6 +1412,7 @@ async def main() -> None:
     async def create(
         self,
         *,
+        annotator_evaluation_enabled: typing.Optional[bool] = OMIT,
         color: typing.Optional[str] = OMIT,
         control_weights: typing.Optional[typing.Optional[typing.Any]] = OMIT,
         created_by: typing.Optional[UserSimpleRequest] = OMIT,
@@ -1434,6 +1449,9 @@ async def create(
 
         Parameters
         ----------
+        annotator_evaluation_enabled : typing.Optional[bool]
+            Enable annotator evaluation for the project
+
         color : typing.Optional[str]
 
         control_weights : typing.Optional[typing.Optional[typing.Any]]
@@ -1543,6 +1561,7 @@ async def main() -> None:
             "api/projects/",
             method="POST",
             json={
+                "annotator_evaluation_enabled": annotator_evaluation_enabled,
                 "color": color,
                 "control_weights": control_weights,
                 "created_by": convert_and_respect_annotation_metadata(
@@ -1822,8 +1841,10 @@ async def update(
         agreement_threshold: typing.Optional[str] = OMIT,
         annotation_limit_count: typing.Optional[int] = OMIT,
         annotation_limit_percent: typing.Optional[str] = OMIT,
+        annotator_evaluation_enabled: typing.Optional[bool] = OMIT,
         annotator_evaluation_minimum_score: typing.Optional[str] = OMIT,
         annotator_evaluation_minimum_tasks: typing.Optional[int] = OMIT,
+        annotator_evaluation_onboarding_tasks: typing.Optional[int] = OMIT,
         assignment_settings: typing.Optional[AssignmentSettingsRequest] = OMIT,
         color: typing.Optional[str] = OMIT,
         comment_classification_config: typing.Optional[str] = OMIT,
@@ -1881,10 +1902,15 @@ async def update(
 
         annotation_limit_percent : typing.Optional[str]
 
+        annotator_evaluation_enabled : typing.Optional[bool]
+            Enable annotator evaluation for the project
+
         annotator_evaluation_minimum_score : typing.Optional[str]
 
         annotator_evaluation_minimum_tasks : typing.Optional[int]
 
+        annotator_evaluation_onboarding_tasks : typing.Optional[int]
+
         assignment_settings : typing.Optional[AssignmentSettingsRequest]
 
         color : typing.Optional[str]
@@ -2022,8 +2048,10 @@ async def main() -> None:
                 "agreement_threshold": agreement_threshold,
                 "annotation_limit_count": annotation_limit_count,
                 "annotation_limit_percent": annotation_limit_percent,
+                "annotator_evaluation_enabled": annotator_evaluation_enabled,
                 "annotator_evaluation_minimum_score": annotator_evaluation_minimum_score,
                 "annotator_evaluation_minimum_tasks": annotator_evaluation_minimum_tasks,
+                "annotator_evaluation_onboarding_tasks": annotator_evaluation_onboarding_tasks,
                 "assignment_settings": convert_and_respect_annotation_metadata(
                     object_=assignment_settings, annotation=AssignmentSettingsRequest, direction="write"
                 ),

src/label_studio_sdk/types/all_roles_project_list.py

@@ -27,8 +27,14 @@ class AllRolesProjectList(UncheckedBaseModel):
     allow_stream: typing.Optional[bool] = None
     annotation_limit_count: typing.Optional[int] = None
     annotation_limit_percent: typing.Optional[str] = None
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     annotator_evaluation_minimum_score: typing.Optional[str] = None
     annotator_evaluation_minimum_tasks: typing.Optional[int] = None
+    annotator_evaluation_onboarding_tasks: typing.Optional[int] = None
     assignment_settings: AssignmentSettings
     blueprints: typing.Optional[typing.List[BlueprintList]] = None
     color: typing.Optional[str] = None

src/label_studio_sdk/types/lse_project.py

@@ -25,8 +25,14 @@ class LseProject(UncheckedBaseModel):
 
     annotation_limit_count: typing.Optional[int] = None
     annotation_limit_percent: typing.Optional[str] = None
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     annotator_evaluation_minimum_score: typing.Optional[str] = None
     annotator_evaluation_minimum_tasks: typing.Optional[int] = None
+    annotator_evaluation_onboarding_tasks: typing.Optional[int] = None
     assignment_settings: AssignmentSettings
     color: typing.Optional[str] = None
     comment_classification_config: typing.Optional[str] = None

src/label_studio_sdk/types/lse_project_create.py

@@ -16,6 +16,11 @@ class LseProjectCreate(UncheckedBaseModel):
     make sure, that you use correct one(Project.objects.with_counts())
     """
 
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     color: typing.Optional[str] = None
     config_has_control_tags: typing.Optional[bool] = pydantic.Field(default=None)
     """

src/label_studio_sdk/types/lse_project_response.py

@@ -25,8 +25,14 @@ class LseProjectResponse(UncheckedBaseModel):
     allow_stream: typing.Optional[bool] = None
     annotation_limit_count: typing.Optional[int] = None
     annotation_limit_percent: typing.Optional[str] = None
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     annotator_evaluation_minimum_score: typing.Optional[str] = None
     annotator_evaluation_minimum_tasks: typing.Optional[int] = None
+    annotator_evaluation_onboarding_tasks: typing.Optional[int] = None
     assignment_settings: AssignmentSettings
     color: typing.Optional[str] = None
     comment_classification_config: typing.Optional[str] = None

src/label_studio_sdk/types/lse_project_update.py

@@ -25,8 +25,14 @@ class LseProjectUpdate(UncheckedBaseModel):
 
     annotation_limit_count: typing.Optional[int] = None
     annotation_limit_percent: typing.Optional[str] = None
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     annotator_evaluation_minimum_score: typing.Optional[str] = None
     annotator_evaluation_minimum_tasks: typing.Optional[int] = None
+    annotator_evaluation_onboarding_tasks: typing.Optional[int] = None
     assignment_settings: AssignmentSettings
     color: typing.Optional[str] = None
     comment_classification_config: typing.Optional[str] = None

src/label_studio_sdk/types/project.py

@@ -16,6 +16,11 @@ class Project(UncheckedBaseModel):
     make sure, that you use correct one(Project.objects.with_counts())
     """
 
+    annotator_evaluation_enabled: typing.Optional[bool] = pydantic.Field(default=None)
+    """
+    Enable annotator evaluation for the project
+    """
+
     color: typing.Optional[str] = None
     config_has_control_tags: typing.Optional[bool] = pydantic.Field(default=None)
     """