From 1b20a11fe6fd3ff01ebe57107e3f32e155d3948e Mon Sep 17 00:00:00 2001 From: tianweidut Date: Fri, 9 Sep 2022 10:49:39 +0800 Subject: [PATCH] refactor pfp code --- client/starwhale/api/_impl/metric.py | 6 +- client/starwhale/api/_impl/model.py | 25 +- client/starwhale/api/_impl/wrapper.py | 13 +- client/starwhale/core/dataset/type.py | 10 +- client/starwhale/core/eval/model.py | 39 +-- client/starwhale/core/eval/view.py | 54 ++- client/tests/core/test_model.py | 1 + example/PennFudanPed/.gitignore | 2 + example/PennFudanPed/Makefile | 12 + example/PennFudanPed/code/coco_utils.py | 259 --------------- example/PennFudanPed/code/ds.py | 75 ----- example/PennFudanPed/code/engine.py | 125 ------- example/PennFudanPed/code/ppl.py | 77 ----- example/PennFudanPed/code/test.py | 58 ---- example/PennFudanPed/code/train.py | 83 ----- example/PennFudanPed/code/transforms.py | 50 --- example/PennFudanPed/config/__init__.py | 0 example/PennFudanPed/config/config.py | 0 example/PennFudanPed/config/hyperparam.json | 0 example/PennFudanPed/dataset.yaml | 10 +- example/PennFudanPed/model.yaml | 10 +- .../PennFudanPed/{code => pfp}/__init__.py | 0 .../{code/data_slicer.py => pfp/dataset.py} | 38 ++- example/PennFudanPed/{code => pfp}/model.py | 29 +- example/PennFudanPed/pfp/ppl.py | 102 ++++++ example/PennFudanPed/pfp/train.py | 309 ++++++++++++++++++ .../{code/utils.py => pfp/utils/__init__.py} | 90 +---- .../{code => pfp/utils}/coco_eval.py | 142 ++++---- example/PennFudanPed/pfp/utils/coco_utils.py | 93 ++++++ example/PennFudanPed/requirements.txt | 4 - example/PennFudanPed/runtime.yaml | 5 - .../runtime/pytorch/requirements-sw-lock.txt | 36 +- example/runtime/pytorch/runtime.yaml | 15 +- 33 files changed, 773 insertions(+), 999 deletions(-) create mode 100644 example/PennFudanPed/.gitignore create mode 100644 example/PennFudanPed/Makefile delete mode 100644 example/PennFudanPed/code/coco_utils.py delete mode 100644 example/PennFudanPed/code/ds.py delete mode 100644 example/PennFudanPed/code/engine.py delete mode 100644 example/PennFudanPed/code/ppl.py delete mode 100644 example/PennFudanPed/code/test.py delete mode 100644 example/PennFudanPed/code/train.py delete mode 100644 example/PennFudanPed/code/transforms.py delete mode 100644 example/PennFudanPed/config/__init__.py delete mode 100644 example/PennFudanPed/config/config.py delete mode 100644 example/PennFudanPed/config/hyperparam.json rename example/PennFudanPed/{code => pfp}/__init__.py (100%) rename example/PennFudanPed/{code/data_slicer.py => pfp/dataset.py} (66%) rename example/PennFudanPed/{code => pfp}/model.py (72%) create mode 100644 example/PennFudanPed/pfp/ppl.py create mode 100644 example/PennFudanPed/pfp/train.py rename example/PennFudanPed/{code/utils.py => pfp/utils/__init__.py} (83%) rename example/PennFudanPed/{code => pfp/utils}/coco_eval.py (75%) create mode 100644 example/PennFudanPed/pfp/utils/coco_utils.py delete mode 100644 example/PennFudanPed/requirements.txt delete mode 100644 example/PennFudanPed/runtime.yaml diff --git a/client/starwhale/api/_impl/metric.py b/client/starwhale/api/_impl/metric.py index 17c41a07fc..db497c4e5f 100644 --- a/client/starwhale/api/_impl/metric.py +++ b/client/starwhale/api/_impl/metric.py @@ -1,6 +1,7 @@ from __future__ import annotations import typing as t +from enum import Enum, unique from functools import wraps from sklearn.metrics import ( # type: ignore @@ -17,7 +18,8 @@ from .model import PipelineHandler -class MetricKind: +@unique +class MetricKind(Enum): MultiClassification = "multi_classification" @@ -40,7 +42,7 @@ def _wrapper(*args: t.Any, **kwargs: t.Any) -> t.Dict[str, t.Any]: else: y_true, y_pred = _rt - _r: t.Dict[str, t.Any] = {"kind": MetricKind.MultiClassification} + _r: t.Dict[str, t.Any] = {"kind": MetricKind.MultiClassification.value} cr = classification_report( y_true, y_pred, output_dict=True, labels=all_labels ) diff --git a/client/starwhale/api/_impl/model.py b/client/starwhale/api/_impl/model.py index ffbc399754..52d1ea6102 100644 --- a/client/starwhale/api/_impl/model.py +++ b/client/starwhale/api/_impl/model.py @@ -1,7 +1,6 @@ from __future__ import annotations import io -import os import sys import math import base64 @@ -22,7 +21,6 @@ from starwhale.utils.fs import ensure_dir, ensure_file from starwhale.base.type import URIType, RunSubDirType from starwhale.utils.log import StreamWrapper -from starwhale.consts.env import SWEnv from starwhale.utils.error import FieldTypeOrValueError from starwhale.api._impl.job import Context from starwhale.core.job.model import STATUS @@ -93,7 +91,9 @@ def __init__( # TODO: split status/result files self._timeline_writer = _jl_writer(self.status_dir / "timeline") - self.evaluation = self._init_datastore() + self.evaluation = Evaluation( + eval_id=self.context.version, project=self.context.project + ) self._monkey_patch() def _init_dir(self) -> None: @@ -108,11 +108,6 @@ def _init_dir(self) -> None: ensure_dir(self.status_dir) ensure_dir(self.log_dir) - def _init_datastore(self) -> Evaluation: - os.environ[SWEnv.project] = self.context.project - os.environ[SWEnv.eval_version] = self.context.version - return Evaluation() - def _init_logger(self) -> t.Tuple[loguru.Logger, loguru.Logger]: # TODO: remove logger first? # TODO: add custom log format, include daemonset pod name @@ -181,11 +176,11 @@ def ppl(self, data: t.Any, **kw: t.Any) -> t.Any: def cmp(self, ppl_result: PPLResultIterator) -> t.Any: raise NotImplementedError - def _builtin_serialize(self, *data: t.Any) -> bytes: + def _builtin_serialize(self, data: t.Any) -> bytes: return dill.dumps(data) # type: ignore - def ppl_result_serialize(self, *data: t.Any) -> bytes: - return self._builtin_serialize(*data) + def ppl_result_serialize(self, data: t.Any) -> bytes: + return self._builtin_serialize(data) def ppl_result_deserialize(self, data: bytes) -> t.Any: return dill.loads(base64.b64decode(data)) @@ -194,7 +189,7 @@ def annotations_serialize(self, data: t.Any) -> bytes: return self._builtin_serialize(data) def annotations_deserialize(self, data: bytes) -> bytes: - return dill.loads(base64.b64decode(data))[0] # type: ignore + return dill.loads(base64.b64decode(data)) # type: ignore def deserialize(self, data: t.Dict[str, t.Any]) -> t.Any: data["result"] = self.ppl_result_deserialize(data["result"]) @@ -275,14 +270,14 @@ def _starwhale_internal_run_ppl(self) -> None: else: exception = None - self._do_record(_idx, _annotations, exception, *pred) + self._do_record(_idx, _annotations, exception, pred) def _do_record( self, idx: int, annotations: t.Dict, exception: t.Optional[Exception], - *args: t.Any, + pred: t.Any, ) -> None: _timeline = { "time": now_str(), @@ -296,7 +291,7 @@ def _do_record( _b64: t.Callable[[bytes], str] = lambda x: base64.b64encode(x).decode("ascii") self.evaluation.log_result( data_id=idx, - result=_b64(self.ppl_result_serialize(*args)), + result=_b64(self.ppl_result_serialize(pred)), annotations=_b64(self.annotations_serialize(annotations)), ) self._update_status(STATUS.RUNNING) diff --git a/client/starwhale/api/_impl/wrapper.py b/client/starwhale/api/_impl/wrapper.py index 285fa362c9..1463cf4428 100644 --- a/client/starwhale/api/_impl/wrapper.py +++ b/client/starwhale/api/_impl/wrapper.py @@ -47,19 +47,20 @@ def _log(self, table_name: str, record: Dict[str, Any]) -> None: class Evaluation(Logger): - def __init__(self, eval_id: Optional[str] = None): - if eval_id is None: - eval_id = os.getenv(SWEnv.eval_version, None) - if eval_id is None: + def __init__(self, eval_id: str = "", project: str = ""): + eval_id = eval_id or os.getenv(SWEnv.eval_version, "") + if not eval_id: raise RuntimeError("eval id should not be None") if re.match(r"^[A-Za-z0-9-_]+$", eval_id) is None: raise RuntimeError( f"invalid eval id {eval_id}, only letters(A-Z, a-z), digits(0-9), hyphen('-'), and underscore('_') are allowed" ) self.eval_id = eval_id - self.project = os.getenv(SWEnv.project) - if self.project is None: + + self.project = project or os.getenv(SWEnv.project, "") + if not self.project: raise RuntimeError(f"{SWEnv.project} is not set") + self._results_table_name = self._get_datastore_table_name("results") self._summary_table_name = f"project/{self.project}/eval/summary" self._init_writers([self._results_table_name, self._summary_table_name]) diff --git a/client/starwhale/core/dataset/type.py b/client/starwhale/core/dataset/type.py index 7a1b80bc3c..1eefb66aa4 100644 --- a/client/starwhale/core/dataset/type.py +++ b/client/starwhale/core/dataset/type.py @@ -2,6 +2,7 @@ import io import os +import base64 import typing as t from abc import ABCMeta, abstractmethod from enum import Enum, unique @@ -166,6 +167,9 @@ class ArtifactType(Enum): Text = "text" +_TBAType = t.TypeVar("_TBAType", bound="BaseArtifact") + + class BaseArtifact(ASDictMixin, metaclass=ABCMeta): def __init__( self, @@ -224,6 +228,10 @@ def to_bytes(self) -> bytes: else: raise NoSupportError(f"read raw for type:{type(self.fp)}") + def carry_raw_data(self: _TBAType) -> _TBAType: + self._raw_base64_data = base64.b64encode(self.to_bytes()).decode() + return self + def astype(self) -> t.Dict[str, t.Any]: return { "type": self.type, @@ -379,7 +387,7 @@ def __init__( image_id: int, category_id: int, segmentation: t.Union[t.List, t.Dict], - area: float, + area: t.Union[float, int], bbox: t.Union[BoundingBox, t.List[float]], iscrowd: int, ) -> None: diff --git a/client/starwhale/core/eval/model.py b/client/starwhale/core/eval/model.py index f5d41f6454..040dc4cdf0 100644 --- a/client/starwhale/core/eval/model.py +++ b/client/starwhale/core/eval/model.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import json import typing as t import subprocess @@ -8,8 +7,6 @@ from http import HTTPStatus from collections import defaultdict -from loguru import logger - from starwhale.utils import load_yaml from starwhale.consts import HTTPMethod, DEFAULT_PAGE_IDX, DEFAULT_PAGE_SIZE from starwhale.base.uri import URI @@ -17,12 +14,12 @@ from starwhale.api._impl import wrapper from starwhale.base.type import InstanceType, JobOperationType from starwhale.base.cloud import CloudRequestMixed -from starwhale.consts.env import SWEnv from starwhale.utils.http import ignore_error from starwhale.utils.error import NotFoundError, NoSupportError from starwhale.utils.config import SWCliConfigMixed from starwhale.utils.process import check_call from starwhale.core.eval.store import EvaluationStorage +from starwhale.api._impl.metric import MetricKind from starwhale.core.eval.executor import EvalExecutor from starwhale.core.runtime.process import Process as RuntimeProcess @@ -95,22 +92,26 @@ def _get_version(self) -> str: raise NotImplementedError def _get_report(self) -> t.Dict[str, t.Any]: - # use datastore - os.environ[SWEnv.project] = self.uri.project - os.environ[SWEnv.eval_version] = self._get_version() - logger.debug( - f"eval instance:{self.uri.instance}, project:{self.uri.project}, eval_id:{self._get_version()}" - ) - _evaluation = wrapper.Evaluation() - _summary = _evaluation.get_metrics() - return dict( - summary=_summary, - labels={str(i): l for i, l in enumerate(list(_evaluation.get("labels")))}, - confusion_matrix=dict( - binarylabel=list(_evaluation.get("confusion_matrix/binarylabel")) - ), - kind=_summary["kind"], + evaluation = wrapper.Evaluation( + eval_id=self._get_version(), project=self.uri.project ) + summary = evaluation.get_metrics() + kind = summary.get("kind", "") + + ret = { + "kind": kind, + "summary": summary, + } + + if kind == MetricKind.MultiClassification.value: + ret["labels"] = { + str(i): l for i, l in enumerate(list(evaluation.get("labels"))) + } + ret["confusion_matrix"] = { + "binarylabel": list(evaluation.get("confusion_matrix/binarylabel")) + } + + return ret @classmethod def _get_job_cls( diff --git a/client/starwhale/core/eval/view.py b/client/starwhale/core/eval/view.py index 9461738b62..77cf52a3cf 100644 --- a/client/starwhale/core/eval/view.py +++ b/client/starwhale/core/eval/view.py @@ -3,9 +3,10 @@ from rich import box from loguru import logger -from rich.tree import Tree +from rich.panel import Panel from rich.table import Table from rich.pretty import Pretty +from rich.columns import Columns from starwhale.utils import Order, console, sort_obj_list from starwhale.consts import ( @@ -18,6 +19,7 @@ from starwhale.base.type import URIType, InstanceType, JobOperationType from starwhale.base.view import BaseTermView from starwhale.core.eval.model import EvaluationJob +from starwhale.api._impl.metric import MetricKind class JobTermView(BaseTermView): @@ -130,7 +132,21 @@ def info(self, page: int = DEFAULT_PAGE_IDX, size: int = DEFAULT_PAGE_SIZE) -> N self._print_tasks(_rt["tasks"][0]) if "report" in _rt: - self._render_job_report(_rt["report"]) + _report = _rt["report"] + _kind = _rt["report"].get("kind", "") + + if "summary" in _report: + self._render_summary_report(_report["summary"], _kind) + + if _kind == MetricKind.MultiClassification.value: + self._render_multi_classification_job_report(_rt["report"]) + + def _render_summary_report(self, summary: t.Dict[str, t.Any], kind: str) -> None: + console.rule(f"[bold green]{kind.upper()} Summary") + contents = [ + Panel(f"[b]{k}[/b]\n[yellow]{v}", expand=True) for k, v in summary.items() + ] + console.print(Columns(contents)) def _print_tasks(self, tasks: t.List[t.Dict[str, t.Any]]) -> None: table = Table(box=box.SIMPLE) @@ -159,8 +175,9 @@ def _print_tasks(self, tasks: t.List[t.Dict[str, t.Any]]) -> None: ) console.print(table) - # TODO: use new result format - def _render_job_report(self, report: t.Dict[str, t.Any]) -> None: + def _render_multi_classification_job_report( + self, report: t.Dict[str, t.Any] + ) -> None: if not report: console.print(":turtle: no report") return @@ -168,30 +185,7 @@ def _render_job_report(self, report: t.Dict[str, t.Any]) -> None: labels: t.Dict[str, t.Any] = report.get("labels", {}) sort_label_names = sorted(list(labels.keys())) - def _print_report() -> None: - # TODO: add other kind report - def _r(_tree: t.Any, _obj: t.Any) -> None: - if not isinstance(_obj, dict): - _tree.add(str(_obj)) - - for _k, _v in _obj.items(): - if _k == "id": - continue - if isinstance(_v, (list, tuple)): - _k = f"{_k}: [green]{'|'.join(_v)}" - elif isinstance(_v, dict): - _k = _k - elif isinstance(_v, str): - _k = f"{_k}:{_v}" - else: - _k = f"{_k}: [green]{_v:.4f}" - - _ntree = _tree.add(_k) - if isinstance(_v, dict): - _r(_ntree, _v) - - tree = Tree("Summary") - _r(tree, report["summary"]) + def _print_labels() -> None: if len(labels) == 0: return @@ -209,7 +203,7 @@ def _r(_tree: t.Any, _obj: t.Any) -> None: ) console.rule(f"[bold green]{report['kind'].upper()} Report") - console.print(self.comparison(tree, table)) + console.print(table) def _print_confusion_matrix() -> None: cm = report.get("confusion_matrix", {}) @@ -236,7 +230,7 @@ def _print_confusion_matrix() -> None: console.rule(f"[bold green]{report['kind'].upper()} Confusion Matrix") console.print(self.comparison(mtable, btable)) - _print_report() + _print_labels() _print_confusion_matrix() @classmethod diff --git a/client/tests/core/test_model.py b/client/tests/core/test_model.py index 36d7d731bb..c2c1d986bd 100644 --- a/client/tests/core/test_model.py +++ b/client/tests/core/test_model.py @@ -210,6 +210,7 @@ def some(self): Context( workdir=Path(_model_data_dir), version="rwerwe9", + project="self", ), "some", ) diff --git a/example/PennFudanPed/.gitignore b/example/PennFudanPed/.gitignore new file mode 100644 index 0000000000..5ac07612fb --- /dev/null +++ b/example/PennFudanPed/.gitignore @@ -0,0 +1,2 @@ +data/ +model/ diff --git a/example/PennFudanPed/Makefile b/example/PennFudanPed/Makefile new file mode 100644 index 0000000000..16dfa57353 --- /dev/null +++ b/example/PennFudanPed/Makefile @@ -0,0 +1,12 @@ +.POHNY: train +train: + mkdir -p models + python3 pfp/train.py + +.POHNY: download-data +download-data: + rm -rf data + mkdir -p data + curl -o data/pfp.zip https://www.cis.upenn.edu/~jshi/ped_html/PennFudanPed.zip + unzip data/pfp.zip -d data + rm -rf data/pfp.zip diff --git a/example/PennFudanPed/code/coco_utils.py b/example/PennFudanPed/code/coco_utils.py deleted file mode 100644 index 64e30afb06..0000000000 --- a/example/PennFudanPed/code/coco_utils.py +++ /dev/null @@ -1,259 +0,0 @@ -import copy -import os - -import torch -import torch.utils.data -import torchvision - -from pycocotools import mask as coco_mask -from pycocotools.coco import COCO - -try: - from . import transforms as T -except ImportError: - import transforms as T - -class FilterAndRemapCocoCategories(object): - def __init__(self, categories, remap=True): - self.categories = categories - self.remap = remap - - def __call__(self, image, target): - anno = target["annotations"] - anno = [obj for obj in anno if obj["category_id"] in self.categories] - if not self.remap: - target["annotations"] = anno - return image, target - anno = copy.deepcopy(anno) - for obj in anno: - obj["category_id"] = self.categories.index(obj["category_id"]) - target["annotations"] = anno - return image, target - - -def convert_coco_poly_to_mask(segmentations, height, width): - masks = [] - for polygons in segmentations: - rles = coco_mask.frPyObjects(polygons, height, width) - mask = coco_mask.decode(rles) - if len(mask.shape) < 3: - mask = mask[..., None] - mask = torch.as_tensor(mask, dtype=torch.uint8) - mask = mask.any(dim=2) - masks.append(mask) - if masks: - masks = torch.stack(masks, dim=0) - else: - masks = torch.zeros((0, height, width), dtype=torch.uint8) - return masks - - -class ConvertCocoPolysToMask(object): - def __call__(self, image, target): - w, h = image.size - - image_id = target["image_id"] - image_id = torch.tensor([image_id]) - - anno = target["annotations"] - - anno = [obj for obj in anno if obj['iscrowd'] == 0] - - boxes = [obj["bbox"] for obj in anno] - # guard against no boxes via resizing - boxes = torch.as_tensor(boxes, dtype=torch.float32).reshape(-1, 4) - boxes[:, 2:] += boxes[:, :2] - boxes[:, 0::2].clamp_(min=0, max=w) - boxes[:, 1::2].clamp_(min=0, max=h) - - classes = [obj["category_id"] for obj in anno] - classes = torch.tensor(classes, dtype=torch.int64) - - segmentations = [obj["segmentation"] for obj in anno] - masks = convert_coco_poly_to_mask(segmentations, h, w) - - keypoints = None - if anno and "keypoints" in anno[0]: - keypoints = [obj["keypoints"] for obj in anno] - keypoints = torch.as_tensor(keypoints, dtype=torch.float32) - num_keypoints = keypoints.shape[0] - if num_keypoints: - keypoints = keypoints.view(num_keypoints, -1, 3) - - keep = (boxes[:, 3] > boxes[:, 1]) & (boxes[:, 2] > boxes[:, 0]) - boxes = boxes[keep] - classes = classes[keep] - masks = masks[keep] - if keypoints is not None: - keypoints = keypoints[keep] - - target = {} - target["boxes"] = boxes - target["labels"] = classes - target["masks"] = masks - target["image_id"] = image_id - if keypoints is not None: - target["keypoints"] = keypoints - - # for conversion to coco api - area = torch.tensor([obj["area"] for obj in anno]) - iscrowd = torch.tensor([obj["iscrowd"] for obj in anno]) - target["area"] = area - target["iscrowd"] = iscrowd - - return image, target - - -def _coco_remove_images_without_annotations(dataset, cat_list=None): - def _has_only_empty_bbox(anno): - return all(any(o <= 1 for o in obj["bbox"][2:]) for obj in anno) - - def _count_visible_keypoints(anno): - return sum(sum(1 for v in ann["keypoints"][2::3] if v > 0) for ann in anno) - - min_keypoints_per_image = 10 - - def _has_valid_annotation(anno): - # if it's empty, there is no annotation - if len(anno) == 0: - return False - # if all boxes have close to zero area, there is no annotation - if _has_only_empty_bbox(anno): - return False - # keypoints task have a slight different critera for considering - # if an annotation is valid - if "keypoints" not in anno[0]: - return True - # for keypoint detection tasks, only consider valid images those - # containing at least min_keypoints_per_image - if _count_visible_keypoints(anno) >= min_keypoints_per_image: - return True - return False - - assert isinstance(dataset, torchvision.datasets.CocoDetection) - ids = [] - for ds_idx, img_id in enumerate(dataset.ids): - ann_ids = dataset.coco.getAnnIds(imgIds=img_id, iscrowd=None) - anno = dataset.coco.loadAnns(ann_ids) - if cat_list: - anno = [obj for obj in anno if obj["category_id"] in cat_list] - if _has_valid_annotation(anno): - ids.append(ds_idx) - - dataset = torch.utils.data.Subset(dataset, ids) - return dataset - - -def convert_to_coco_api(ds): - coco_ds = COCO() - # annotation IDs need to start at 1, not 0, see torchvision issue #1530 - ann_id = 1 - dataset = {'images': [], 'categories': [], 'annotations': []} - categories = set() - img_idx = 0 - for img, targets in ds: - # find better way to get target - # targets = ds.get_annotations(img_idx) - # img, targets = ds[img_idx] - image_id = targets["image_id"].item() - img_dict = {} - img_dict['id'] = image_id - if isinstance(img, torch.Tensor): - img_dict['height'] = img.shape[-2] - img_dict['width'] = img.shape[-1] - else: - img_dict['height'] = img['height'] - img_dict['width'] = img['width'] - dataset['images'].append(img_dict) - bboxes = targets["boxes"] - bboxes[:, 2:] -= bboxes[:, :2] - bboxes = bboxes.tolist() - labels = targets['labels'].tolist() - areas = targets['area'].tolist() - iscrowd = targets['iscrowd'].tolist() - if 'masks' in targets: - masks = targets['masks'] - # make masks Fortran contiguous for coco_mask - masks = masks.permute(0, 2, 1).contiguous().permute(0, 2, 1) - if 'keypoints' in targets: - keypoints = targets['keypoints'] - keypoints = keypoints.reshape(keypoints.shape[0], -1).tolist() - num_objs = len(bboxes) - for i in range(num_objs): - ann = {} - ann['image_id'] = image_id - ann['bbox'] = bboxes[i] - ann['category_id'] = labels[i] - categories.add(labels[i]) - ann['area'] = areas[i] - ann['iscrowd'] = iscrowd[i] - ann['id'] = ann_id - if 'masks' in targets: - ann["segmentation"] = coco_mask.encode(masks[i].numpy()) - if 'keypoints' in targets: - ann['keypoints'] = keypoints[i] - ann['num_keypoints'] = sum(k != 0 for k in keypoints[i][2::3]) - dataset['annotations'].append(ann) - ann_id += 1 - img_idx += 1 - dataset['categories'] = [{'id': i} for i in sorted(categories)] - coco_ds.dataset = dataset - coco_ds.createIndex() - return coco_ds - - -def get_coco_api_from_dataset(dataset): - for _ in range(10): - if isinstance(dataset, torchvision.datasets.CocoDetection): - break - if isinstance(dataset, torch.utils.data.Subset): - dataset = dataset.dataset - if isinstance(dataset, torchvision.datasets.CocoDetection): - return dataset.coco - return convert_to_coco_api(dataset) - - -class CocoDetection(torchvision.datasets.CocoDetection): - def __init__(self, img_folder, ann_file, transforms): - super(CocoDetection, self).__init__(img_folder, ann_file) - self._transforms = transforms - - def __getitem__(self, idx): - img, target = super(CocoDetection, self).__getitem__(idx) - image_id = self.ids[idx] - target = dict(image_id=image_id, annotations=target) - if self._transforms is not None: - img, target = self._transforms(img, target) - return img, target - - -def get_coco(root, image_set, transforms, mode='instances'): - anno_file_template = "{}_{}2017.json" - PATHS = { - "train": ("train2017", os.path.join("annotations", anno_file_template.format(mode, "train"))), - "val": ("val2017", os.path.join("annotations", anno_file_template.format(mode, "val"))), - # "train": ("val2017", os.path.join("annotations", anno_file_template.format(mode, "val"))) - } - - t = [ConvertCocoPolysToMask()] - - if transforms is not None: - t.append(transforms) - transforms = T.Compose(t) - - img_folder, ann_file = PATHS[image_set] - img_folder = os.path.join(root, img_folder) - ann_file = os.path.join(root, ann_file) - - dataset = CocoDetection(img_folder, ann_file, transforms=transforms) - - if image_set == "train": - dataset = _coco_remove_images_without_annotations(dataset) - - # dataset = torch.utils.data.Subset(dataset, [i for i in range(500)]) - - return dataset - - -def get_coco_kp(root, image_set, transforms): - return get_coco(root, image_set, transforms, mode="person_keypoints") diff --git a/example/PennFudanPed/code/ds.py b/example/PennFudanPed/code/ds.py deleted file mode 100644 index 5c4791dcce..0000000000 --- a/example/PennFudanPed/code/ds.py +++ /dev/null @@ -1,75 +0,0 @@ -import os - -import numpy as np -import torch -from PIL import Image - - -def mask_to_coco_target(mask_img, img_idx): - mask = np.array(mask_img) - # instances are encoded as different colors - obj_ids = np.unique(mask) - # first id is the background, so remove it - obj_ids = obj_ids[1:] - - # split the color-encoded mask into a set - # of binary masks - masks = mask == obj_ids[:, None, None] - - # get bounding box coordinates for each mask - num_objs = len(obj_ids) - boxes = [] - for i in range(num_objs): - pos = np.where(masks[i]) - xmin = np.min(pos[1]) - xmax = np.max(pos[1]) - ymin = np.min(pos[0]) - ymax = np.max(pos[0]) - boxes.append([xmin, ymin, xmax, ymax]) - - boxes = torch.as_tensor(boxes, dtype=torch.float32) - # there is only one class - labels = torch.ones((num_objs,), dtype=torch.int64) - masks = torch.as_tensor(masks, dtype=torch.uint8) - - image_id = torch.tensor([img_idx]) - area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0]) - # suppose all instances are not crowd - iscrowd = torch.zeros((num_objs,), dtype=torch.int64) - - target = {} - target["boxes"] = boxes - target["labels"] = labels - target["masks"] = masks - target["image_id"] = image_id - target["area"] = area - target["iscrowd"] = iscrowd - return target - - -class PennFudanDataset: - def __init__(self, root, transforms): - self.root = root - self.transforms = transforms - # load all image files, sorting them to - # ensure that they are aligned - self.imgs = list(sorted(os.listdir(os.path.join(root, "PNGImages")))) - self.masks = list(sorted(os.listdir(os.path.join(root, "PedMasks")))) - - def __getitem__(self, idx): - # load images and masks - img_path = os.path.join(self.root, "PNGImages", self.imgs[idx]) - mask_path = os.path.join(self.root, "PedMasks", self.masks[idx]) - img = Image.open(img_path).convert("RGB") - # note that we haven't converted the mask to RGB, - # because each color corresponds to a different instance - # with 0 being background - mask = Image.open(mask_path) - target = mask_to_coco_target(mask, idx) - if self.transforms is not None: - img, target = self.transforms(img, target) - - return img, target - - def __len__(self): - return len(self.imgs) diff --git a/example/PennFudanPed/code/engine.py b/example/PennFudanPed/code/engine.py deleted file mode 100644 index 53a698d297..0000000000 --- a/example/PennFudanPed/code/engine.py +++ /dev/null @@ -1,125 +0,0 @@ -import math -import sys -import time -import torch - -import torchvision.models.detection.mask_rcnn - -try: - from . import coco_utils -except ImportError: - import coco_utils - -try: - from . import coco_eval -except ImportError: - import coco_eval - -try: - from . import utils as myutils -except ImportError: - import utils as myutils - - -def train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq): - model.train() - metric_logger = myutils.MetricLogger(delimiter=" ") - metric_logger.add_meter('lr', myutils.SmoothedValue(window_size=1, - fmt='{value:.6f}')) - header = 'Epoch: [{}]'.format(epoch) - - lr_scheduler = None - if epoch == 0: - warmup_factor = 1. / 1000 - warmup_iters = min(1000, len(data_loader) - 1) - - lr_scheduler = myutils.warmup_lr_scheduler(optimizer, warmup_iters, - warmup_factor) - - for images, targets in metric_logger.log_every(data_loader, print_freq, - header): - images = list(image.to(device) for image in images) - targets = [{k: v.to(device) for k, v in t.items()} for t in targets] - - loss_dict = model(images, targets) - - losses = sum(loss for loss in loss_dict.values()) - - # reduce losses over all GPUs for logging purposes - loss_dict_reduced = myutils.reduce_dict(loss_dict) - losses_reduced = sum(loss for loss in loss_dict_reduced.values()) - - loss_value = losses_reduced.item() - - if not math.isfinite(loss_value): - print("Loss is {}, stopping training".format(loss_value)) - print(loss_dict_reduced) - sys.exit(1) - - optimizer.zero_grad() - losses.backward() - optimizer.step() - - if lr_scheduler is not None: - lr_scheduler.step() - - metric_logger.update(loss=losses_reduced, **loss_dict_reduced) - metric_logger.update(lr=optimizer.param_groups[0]["lr"]) - - return metric_logger - - -def _get_iou_types(model): - model_without_ddp = model - if isinstance(model, torch.nn.parallel.DistributedDataParallel): - model_without_ddp = model.module - iou_types = ["bbox"] - if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): - iou_types.append("segm") - if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): - iou_types.append("keypoints") - return iou_types - - -@torch.no_grad() -def evaluate(model, data_loader, device): - n_threads = torch.get_num_threads() - # FIXME remove this and make paste_masks_in_image run on the GPU - torch.set_num_threads(1) - cpu_device = torch.device("cpu") - model.eval() - metric_logger = myutils.MetricLogger(delimiter=" ") - header = 'Test:' - - coco = coco_utils.get_coco_api_from_dataset(data_loader.dataset) - iou_types = _get_iou_types(model) - coco_evaluator = coco_eval.CocoEvaluator(coco, iou_types) - - for images, targets in metric_logger.log_every(data_loader, 100, header): - images = list(img.to(device) for img in images) - - torch.cuda.synchronize() - model_time = time.time() - outputs = model(images) - - outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] - model_time = time.time() - model_time - - res = {target["image_id"].item(): output for target, output in - zip(targets, outputs)} - evaluator_time = time.time() - coco_evaluator.update(res) - evaluator_time = time.time() - evaluator_time - metric_logger.update(model_time=model_time, - evaluator_time=evaluator_time) - - # gather the stats from all processes - metric_logger.synchronize_between_processes() - print("Averaged stats:", metric_logger) - coco_evaluator.synchronize_between_processes() - - # accumulate predictions from all images - coco_evaluator.accumulate() - coco_evaluator.summarize() - torch.set_num_threads(n_threads) - return coco_evaluator diff --git a/example/PennFudanPed/code/ppl.py b/example/PennFudanPed/code/ppl.py deleted file mode 100644 index 8b171e95c1..0000000000 --- a/example/PennFudanPed/code/ppl.py +++ /dev/null @@ -1,77 +0,0 @@ -import io -import os -import pickle - -import torch -from PIL import Image -from torchvision.transforms import functional as F - -from starwhale.api.job import Context -from starwhale.api.model import PipelineHandler - -from . import model as mask_rcnn_model -from . import coco_eval, coco_utils - -_ROOT_DIR = os.path.dirname(os.path.dirname(__file__)) - - -class MARSKRCNN(PipelineHandler): - def __init__(self, context: Context) -> None: - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - super().__init__(context=context) - - @torch.no_grad() - def ppl(self, data, **kw): - model = self._load_model(self.device) - files_bytes = pickle.loads(data) - _result = [] - cpu_device = torch.device("cpu") - for file_bytes in files_bytes: - image = Image.open(io.BytesIO(file_bytes.content_bytes)) - _image = F.to_tensor(image) - outputs = model([_image.to(self.device)]) - output = outputs[0] - # [{'boxes':tensor[[],[]]},'labels':tensor[[],[]],'masks':tensor[[[]]]}] - output = {k: v.to(cpu_device) for k, v in output.items()} - output["height"] = _image.shape[-2] - output["width"] = _image.shape[-1] - _result.append(output) - return _result - - def cmp(self, ppl_result): - result, label = [], [] - for _data in ppl_result: - label.append(_data["annotations"]) - (result) = _data["result"] - result.extend(result) - ds = zip(result, label) - coco_ds = coco_utils.convert_to_coco_api(ds) - coco_evaluator = coco_eval.CocoEvaluator(coco_ds, ["bbox", "segm"]) - for outputs, targets in zip(result, label): - res = {targets["image_id"].item(): outputs} - coco_evaluator.update(res) - - # gather the stats from all processes - coco_evaluator.synchronize_between_processes() - - # accumulate predictions from all images - coco_evaluator.accumulate() - coco_evaluator.summarize() - - return { - iou_type: coco_eval.stats.tolist() - for iou_type, coco_eval in coco_evaluator.coco_eval.items() - } - - def _pre(self, input: bytes): - image = Image.open(io.BytesIO(input)) - image = F.to_tensor(image) - return [image.to(self.device)] - - def _load_model(self, device): - s = _ROOT_DIR + "/models/maskrcnn.pth" - net = mask_rcnn_model.get_model_instance_segmentation(2, False, torch.load(s)) - net = net.to(device) - net.eval() - print("mask rcnn model loaded, start to inference...") - return net diff --git a/example/PennFudanPed/code/test.py b/example/PennFudanPed/code/test.py deleted file mode 100644 index d80bca465b..0000000000 --- a/example/PennFudanPed/code/test.py +++ /dev/null @@ -1,58 +0,0 @@ -from pathlib import Path -import os - -import torch -from model import get_model_instance_segmentation -from ds import PennFudanDataset -from train import get_transform -import utils -import coco_utils -import coco_eval - -_ROOT_DIR = os.path.dirname(os.path.dirname(__file__)) -_MODEL_PATH = os.path.join(_ROOT_DIR, "../models/maskrcnn.pth") -_DATA_PATH = os.path.join(_ROOT_DIR, "../data/PennFudanPed") - - -def _load_model( device): - model = get_model_instance_segmentation(2, False, torch.load(_MODEL_PATH)) - model = model.to(device) - model.eval() - print("mask rcnn model loaded, start to inference...") - return model - - -@torch.no_grad() -def main(): - dataset_test = PennFudanDataset(_DATA_PATH, get_transform(train=False)) - indices = torch.randperm(len(dataset_test)).tolist() - dataset_test = torch.utils.data.Subset(dataset_test, indices[-1:]) - data_loader_test = torch.utils.data.DataLoader( - dataset_test, batch_size=1, shuffle=False, num_workers=4, - collate_fn=utils.collate_fn) - device = torch.device('cuda') - model = _load_model(device) - cpu_device = torch.device("cpu") - coco = coco_utils.get_coco_api_from_dataset(data_loader_test.dataset) - coco_evaluator = coco_eval.CocoEvaluator(coco, ["bbox", "segm"]) - for images, targets in data_loader_test: - images = list(img.to(device) for img in images) - torch.cuda.synchronize() - outputs = model(images) - outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] - res = {target["image_id"].item(): output for target, output in - zip(targets, outputs)} - print(res) - coco_evaluator.update(res) - - # gather the stats from all processes - coco_evaluator.synchronize_between_processes() - - # accumulate predictions from all images - coco_evaluator.accumulate() - coco_evaluator.summarize() - result = [{iou_type: coco_eval.stats for iou_type, coco_eval in coco_evaluator.coco_eval.items()}] - print(result) - -if __name__ == "__main__": - main() diff --git a/example/PennFudanPed/code/train.py b/example/PennFudanPed/code/train.py deleted file mode 100644 index 1b50491b75..0000000000 --- a/example/PennFudanPed/code/train.py +++ /dev/null @@ -1,83 +0,0 @@ -# Sample code from the TorchVision 0.3 Object Detection Finetuning Tutorial -# http://pytorch.org/tutorials/intermediate/torchvision_tutorial.html - -import os -import torch - -from engine import train_one_epoch, evaluate -from ds import PennFudanDataset -from model import get_model_instance_segmentation -import utils -import transforms as T - -_ROOT_DIR = os.path.dirname(os.path.dirname(__file__)) -_MODEL_PATH = os.path.join(_ROOT_DIR, "../models/mcrnn.pth") -_DATA_PATH = os.path.join(_ROOT_DIR, "../data/PennFudanPed") - - -def get_transform(train): - transforms = [] - transforms.append(T.ToTensor()) - if train: - transforms.append(T.RandomHorizontalFlip(0.5)) - return T.Compose(transforms) - - -def main(): - # train on the GPU or on the CPU, if a GPU is not available - device = torch.device( - 'cuda') if torch.cuda.is_available() else torch.device('cpu') - - # our dataset has two classes only - background and person - num_classes = 2 - # use our dataset and defined transformations - dataset = PennFudanDataset(_DATA_PATH, get_transform(train=True)) - dataset_test = PennFudanDataset(_DATA_PATH, get_transform(train=False)) - - # split the dataset in train and test set - indices = torch.randperm(len(dataset)).tolist() - dataset = torch.utils.data.Subset(dataset, indices[:-50]) - dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:]) - - # define training and validation data loaders - data_loader = torch.utils.data.DataLoader( - dataset, batch_size=2, shuffle=True, num_workers=4, - collate_fn=utils.collate_fn) - - data_loader_test = torch.utils.data.DataLoader( - dataset_test, batch_size=1, shuffle=False, num_workers=4, - collate_fn=utils.collate_fn) - - # get the model using our helper function - model = get_model_instance_segmentation(num_classes) - - # move model to the right device - model.to(device) - - # construct an optimizer - params = [p for p in model.parameters() if p.requires_grad] - optimizer = torch.optim.SGD(params, lr=0.005, - momentum=0.9, weight_decay=0.0005) - # and a learning rate scheduler - lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, - step_size=3, - gamma=0.1) - - # let's train it for 10 epochs - num_epochs = 10 - - for epoch in range(num_epochs): - # train for one epoch, printing every 10 iterations - train_one_epoch(model, optimizer, data_loader, device, epoch, - print_freq=10) - # update the learning rate - lr_scheduler.step() - # evaluate on the test dataset - evaluate(model, data_loader_test, device=device) - - torch.save(model.state_dict(), _MODEL_PATH) - print("That's it!") - - -if __name__ == "__main__": - main() diff --git a/example/PennFudanPed/code/transforms.py b/example/PennFudanPed/code/transforms.py deleted file mode 100644 index 1eea62e639..0000000000 --- a/example/PennFudanPed/code/transforms.py +++ /dev/null @@ -1,50 +0,0 @@ -import random -import torch - -from torchvision.transforms import functional as F - - -def _flip_coco_person_keypoints(kps, width): - flip_inds = [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15] - flipped_data = kps[:, flip_inds] - flipped_data[..., 0] = width - flipped_data[..., 0] - # Maintain COCO convention that if visibility == 0, then x, y = 0 - inds = flipped_data[..., 2] == 0 - flipped_data[inds] = 0 - return flipped_data - - -class Compose(object): - def __init__(self, transforms): - self.transforms = transforms - - def __call__(self, image, target): - for t in self.transforms: - image, target = t(image, target) - return image, target - - -class RandomHorizontalFlip(object): - def __init__(self, prob): - self.prob = prob - - def __call__(self, image, target): - if random.random() < self.prob: - height, width = image.shape[-2:] - image = image.flip(-1) - bbox = target["boxes"] - bbox[:, [0, 2]] = width - bbox[:, [2, 0]] - target["boxes"] = bbox - if "masks" in target: - target["masks"] = target["masks"].flip(-1) - if "keypoints" in target: - keypoints = target["keypoints"] - keypoints = _flip_coco_person_keypoints(keypoints, width) - target["keypoints"] = keypoints - return image, target - - -class ToTensor(object): - def __call__(self, image, target): - image = F.to_tensor(image) - return image, target \ No newline at end of file diff --git a/example/PennFudanPed/config/__init__.py b/example/PennFudanPed/config/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/example/PennFudanPed/config/config.py b/example/PennFudanPed/config/config.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/example/PennFudanPed/config/hyperparam.json b/example/PennFudanPed/config/hyperparam.json deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/example/PennFudanPed/dataset.yaml b/example/PennFudanPed/dataset.yaml index 308449f6f9..b082d8df20 100644 --- a/example/PennFudanPed/dataset.yaml +++ b/example/PennFudanPed/dataset.yaml @@ -1,11 +1,7 @@ -name: penn_fudan_ped - -process: code.data_slicer:PennFudanPedSlicer +name: pfp +process: pfp.dataset:PFPDatasetBuildExecutor desc: PennFudanPed data and label test dataset -tag: - - bin - attr: alignment_size: 4k - volume_size: 2M + volume_size: 10M diff --git a/example/PennFudanPed/model.yaml b/example/PennFudanPed/model.yaml index b0bb167ea1..088d3c258b 100644 --- a/example/PennFudanPed/model.yaml +++ b/example/PennFudanPed/model.yaml @@ -1,13 +1,7 @@ version: 1.0 name: mask_rcnn - model: - - models/maskrcnn.pth - + - models/mcrnn.pth run: - ppl: code.ppl:MARSKRCNN - + ppl: pfp.ppl:MaskRCnn desc: mask rcnn resnet50 by pytorch - -tag: - - instance segmentation & object dectection \ No newline at end of file diff --git a/example/PennFudanPed/code/__init__.py b/example/PennFudanPed/pfp/__init__.py similarity index 100% rename from example/PennFudanPed/code/__init__.py rename to example/PennFudanPed/pfp/__init__.py diff --git a/example/PennFudanPed/code/data_slicer.py b/example/PennFudanPed/pfp/dataset.py similarity index 66% rename from example/PennFudanPed/code/data_slicer.py rename to example/PennFudanPed/pfp/dataset.py index 5c1e2d62f7..81770dbd47 100644 --- a/example/PennFudanPed/code/data_slicer.py +++ b/example/PennFudanPed/pfp/dataset.py @@ -15,22 +15,33 @@ ) -class PennFudanPedSlicer(BuildExecutor): +class PFPDatasetBuildExecutor(BuildExecutor): def iter_item(self) -> t.Generator[t.Tuple[t.Any, t.Any], None, None]: root_dir = Path(__file__).parent.parent / "data" / "PennFudanPed" names = [p.stem for p in (root_dir / "PNGImages").iterdir()] - for idx, name in enumerate(names): + self.object_id = 1 + for idx, name in enumerate(sorted(names)): data_fpath = root_dir / "PNGImages" / f"{name}.png" mask_fpath = root_dir / "PedMasks" / f"{name}_mask.png" height, width = self._get_image_shape(data_fpath) coco_annotations = self._make_coco_annotations(mask_fpath, idx) annotations = { - "mask": Image(mask_fpath, display_name=name, mime_type=MIMEType.PNG), - "image": {"id": idx, "height": height, "width": width}, + "mask": Image( + mask_fpath, + display_name=name, + mime_type=MIMEType.PNG, + shape=(height, width, 3), + ).carry_raw_data(), + "image": {"id": idx, "height": height, "width": width, "name": name}, "object_nums": len(coco_annotations), "annotations": coco_annotations, } - data = Image(data_fpath, display_name=name, mime_type=MIMEType.PNG) + data = Image( + data_fpath, + display_name=name, + mime_type=MIMEType.PNG, + shape=(height, width, 3), + ) yield data, annotations def _get_image_shape(self, fpath: Path) -> t.Tuple[int, int]: @@ -45,7 +56,6 @@ def _make_coco_annotations( mask = np.array(mask_img) object_ids = np.unique(mask)[1:] binary_mask = mask == object_ids[:, None, None] - objects_num = len(object_ids) # TODO: tune permute without pytorch binary_mask_tensor = torch.as_tensor(binary_mask, dtype=torch.uint8) binary_mask_tensor = ( @@ -55,22 +65,26 @@ def _make_coco_annotations( coco_annotations = [] for i in range(0, len(object_ids)): _pos = np.where(binary_mask[i]) - _xmin, _ymin = np.min(_pos[1]), np.min(_pos[0]) - _xmax, _ymax = np.max(_pos[1]), np.max(_pos[0]) + _xmin, _ymin = float(np.min(_pos[1])), float(np.min(_pos[0])) + _xmax, _ymax = float(np.max(_pos[1])), float(np.max(_pos[0])) _bbox = BoundingBox( x=_xmin, y=_ymin, width=_xmax - _xmin, height=_ymax - _ymin ) + rle: t.Dict = coco_mask.encode(binary_mask_tensor[i].numpy()) # type: ignore + rle["counts"] = rle["counts"].decode("utf-8") + coco_annotations.append( COCOObjectAnnotation( - id=i, + id=self.object_id, image_id=image_id, - category_id=objects_num, - segmentation=coco_mask.encode(binary_mask_tensor[i].numpy()), # type: ignore + category_id=1, # PennFudan Dataset only has one class-PASPersonStanding + segmentation=rle, area=_bbox.width * _bbox.height, bbox=_bbox, - iscrowd=0 if objects_num == 1 else 1, + iscrowd=0, # suppose all instances are not crowd ) ) + self.object_id += 1 return coco_annotations diff --git a/example/PennFudanPed/code/model.py b/example/PennFudanPed/pfp/model.py similarity index 72% rename from example/PennFudanPed/code/model.py rename to example/PennFudanPed/pfp/model.py index 1ac379ec88..9b89b6c3bf 100644 --- a/example/PennFudanPed/code/model.py +++ b/example/PennFudanPed/pfp/model.py @@ -1,12 +1,19 @@ -import torchvision -from torchvision.models.detection.faster_rcnn import FastRCNNPredictor -from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor +from torchvision.models import ResNet50_Weights from torchvision.ops.misc import FrozenBatchNorm2d +from torchvision.models.detection import maskrcnn_resnet50_fpn +from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor +from torchvision.models.detection.faster_rcnn import FastRCNNPredictor -def get_model_instance_segmentation(num_classes, remote_pretrained=True, local_dict=None): +def pretrained_model( + num_classes, + weights=None, + model_local_dict=None, + weights_backbone=None, +): # load an instance segmentation model pre-trained pre-trained on COCO - model = torchvision.models.detection.maskrcnn_resnet50_fpn(pretrained=remote_pretrained) + # model = maskrcnn_resnet50_fpn(weights=weights) + model = maskrcnn_resnet50_fpn(weights=weights, weights_backbone=weights_backbone) # get number of input features for the classifier in_features = model.roi_heads.box_predictor.cls_score.in_features @@ -17,11 +24,12 @@ def get_model_instance_segmentation(num_classes, remote_pretrained=True, local_d in_features_mask = model.roi_heads.mask_predictor.conv5_mask.in_channels hidden_layer = 256 # and replace the mask predictor with a new one - model.roi_heads.mask_predictor = MaskRCNNPredictor(in_features_mask, - hidden_layer, - num_classes) - if not remote_pretrained: - model.load_state_dict(local_dict) + model.roi_heads.mask_predictor = MaskRCNNPredictor( + in_features_mask, hidden_layer, num_classes + ) + + if model_local_dict: + model.load_state_dict(model_local_dict) overwrite_eps(model, 0.0) return model @@ -41,4 +49,3 @@ def overwrite_eps(model, eps: float) -> None: for module in model.modules(): if isinstance(module, FrozenBatchNorm2d): module.eps = eps - diff --git a/example/PennFudanPed/pfp/ppl.py b/example/PennFudanPed/pfp/ppl.py new file mode 100644 index 0000000000..19c291dbc8 --- /dev/null +++ b/example/PennFudanPed/pfp/ppl.py @@ -0,0 +1,102 @@ +import io +import typing as t + +import torch +from PIL import Image as PILImage +from pycocotools.coco import COCO +from torchvision.transforms import functional + +from starwhale.api.job import Context +from starwhale.api.model import PipelineHandler +from starwhale.api.dataset import Image + +from .model import pretrained_model +from .utils import get_model_path +from .utils.coco_eval import CocoEvaluator + + +class MaskRCnn(PipelineHandler): + def __init__(self, context: Context) -> None: + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.model = self._load_model(self.device) + super().__init__(context=context) + + @torch.no_grad() + def ppl(self, img: Image, index: int, **kw): + _img = PILImage.open(io.BytesIO(img.to_bytes())).convert("RGB") + _tensor = functional.to_tensor(_img).to(self.device) + output = self.model(torch.stack([_tensor])) + return index, output[0] + + def cmp(self, ppl_result): + pred_results, annotations = [], [] + for _data in ppl_result: + annotations.append(_data["annotations"]) + pred_results.append(_data["result"]) + + evaluator = make_coco_evaluator(annotations, iou_types=["bbox", "segm"]) + for index, pred in pred_results: + evaluator.update({index: pred}) + + evaluator.synchronize_between_processes() + evaluator.accumulate() + evaluator.summarize() + + detector_metrics_map = [ + "average_precision", + "average_precision_iou50", + "average_precision_iou75", + "ap_across_scales_small", + "ap_across_scales_medium", + "ap_across_scales_large", + "average_recall_max1", + "average_recall_max10", + "average_recall_max100", + "ar_across_scales_small", + "ar_across_scales_medium", + "ar_across_scales_large", + ] + + report = {"kind": "coco_object_detection", "bbox": {}, "segm": {}} + for _iou, _eval in evaluator.coco_eval.items(): + if _iou not in report: + continue + + _stats = _eval.stats.tolist() + for _idx, _label in enumerate(detector_metrics_map): + report[_iou][_label] = _stats[_idx] + + self.evaluation.log_metrics(report) + + def _load_model(self, device): + net = pretrained_model( + 2, + model_local_dict=torch.load(get_model_path(), map_location=device), + ) + net = net.to(device) + net.eval() + print("mask rcnn model loaded, start to inference...") + return net + + +def make_coco_evaluator( + ann_list: t.List[t.Dict], iou_types: t.List[str] +) -> CocoEvaluator: + images = [] + categories = set() + annotations = [] + for _anno in ann_list: + images.append(_anno["image"]) + for _a in _anno["annotations"]: + categories.add(_a["category_id"]) + annotations.append(_a) + + coco = COCO() + coco.dataset = { + "images": images, + "annotations": annotations, + "categories": [{"id": _c} for _c in sorted(categories)], + } + coco.createIndex() + coco_evaluator = CocoEvaluator(coco, iou_types=iou_types) + return coco_evaluator diff --git a/example/PennFudanPed/pfp/train.py b/example/PennFudanPed/pfp/train.py new file mode 100644 index 0000000000..064ca00921 --- /dev/null +++ b/example/PennFudanPed/pfp/train.py @@ -0,0 +1,309 @@ +# Sample code from the TorchVision 0.3 Object Detection Finetuning Tutorial +# http://pytorch.org/tutorials/intermediate/torchvision_tutorial.html + +import os +import sys +import math +import time +import random + +import numpy as np +import torch +import torchvision.models.detection.mask_rcnn +from PIL import Image +from model import pretrained_model +from utils import ( + collate_fn, + reduce_dict, + MetricLogger, + get_data_path, + SmoothedValue, + get_model_path, + warmup_lr_scheduler, +) +from utils.coco_eval import CocoEvaluator +from utils.coco_utils import get_coco_api_from_dataset +from torchvision.models import ResNet50_Weights +from torchvision.transforms import functional +from torchvision.models.detection import MaskRCNN_ResNet50_FPN_Weights + + +class Compose: + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, image, target): + for t in self.transforms: + image, target = t(image, target) + return image, target + + +def _flip_coco_person_keypoints(kps, width): + flip_inds = [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15] + flipped_data = kps[:, flip_inds] + flipped_data[..., 0] = width - flipped_data[..., 0] + # Maintain COCO convention that if visibility == 0, then x, y = 0 + inds = flipped_data[..., 2] == 0 + flipped_data[inds] = 0 + return flipped_data + + +class RandomHorizontalFlip: + def __init__(self, prob): + self.prob = prob + + def __call__(self, image, target): + if random.random() < self.prob: + _, width = image.shape[-2:] + image = image.flip(-1) + bbox = target["boxes"] + bbox[:, [0, 2]] = width - bbox[:, [2, 0]] + target["boxes"] = bbox + if "masks" in target: + target["masks"] = target["masks"].flip(-1) + if "keypoints" in target: + keypoints = target["keypoints"] + keypoints = _flip_coco_person_keypoints(keypoints, width) + target["keypoints"] = keypoints + return image, target + + +class ToTensor(object): + def __call__(self, image, target): + image = functional.to_tensor(image) + return image, target + + +class PennFudanDataset: + def __init__(self, root, transforms): + self.root = root + self.transforms = transforms + # load all image files, sorting them to + # ensure that they are aligned + self.imgs = list(sorted(os.listdir(os.path.join(root, "PNGImages")))) + self.masks = list(sorted(os.listdir(os.path.join(root, "PedMasks")))) + + def __getitem__(self, idx): + # load images and masks + img_path = os.path.join(self.root, "PNGImages", self.imgs[idx]) + mask_path = os.path.join(self.root, "PedMasks", self.masks[idx]) + img = Image.open(img_path).convert("RGB") + # note that we haven't converted the mask to RGB, + # because each color corresponds to a different instance + # with 0 being background + mask = Image.open(mask_path) + target = mask_to_coco_target(mask, idx) + if self.transforms is not None: + img, target = self.transforms(img, target) + + return img, target + + def __len__(self): + return len(self.imgs) + + +def mask_to_coco_target(mask_img, img_idx): + mask = np.array(mask_img) + # instances are encoded as different colors + obj_ids = np.unique(mask) + # first id is the background, so remove it + obj_ids = obj_ids[1:] + + # split the color-encoded mask into a set + # of binary masks + masks = mask == obj_ids[:, None, None] + + # get bounding box coordinates for each mask + num_objs = len(obj_ids) + boxes = [] + for i in range(num_objs): + pos = np.where(masks[i]) + xmin = np.min(pos[1]) + xmax = np.max(pos[1]) + ymin = np.min(pos[0]) + ymax = np.max(pos[0]) + boxes.append([xmin, ymin, xmax, ymax]) + + boxes = torch.as_tensor(boxes, dtype=torch.float32) + # there is only one class + labels = torch.ones((num_objs,), dtype=torch.int64) + masks = torch.as_tensor(masks, dtype=torch.uint8) + + image_id = torch.tensor([img_idx]) + area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0]) + # suppose all instances are not crowd + iscrowd = torch.zeros((num_objs,), dtype=torch.int64) + + target = {} + target["boxes"] = boxes + target["labels"] = labels + target["masks"] = masks + target["image_id"] = image_id + target["area"] = area + target["iscrowd"] = iscrowd + return target + + +def train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq): + model.train() + metric_logger = MetricLogger(delimiter=" ") + metric_logger.add_meter("lr", SmoothedValue(window_size=1, fmt="{value:.6f}")) + header = "Epoch: [{}]".format(epoch) + + lr_scheduler = None + if epoch == 0: + warmup_factor = 1.0 / 1000 + warmup_iters = min(1000, len(data_loader) - 1) + + lr_scheduler = warmup_lr_scheduler(optimizer, warmup_iters, warmup_factor) + + for images, targets in metric_logger.log_every(data_loader, print_freq, header): + images = list(image.to(device) for image in images) + targets = [{k: v.to(device) for k, v in t.items()} for t in targets] + + loss_dict = model(images, targets) + + losses = sum(loss for loss in loss_dict.values()) + + # reduce losses over all GPUs for logging purposes + loss_dict_reduced = reduce_dict(loss_dict) + losses_reduced = sum(loss for loss in loss_dict_reduced.values()) + + loss_value = losses_reduced.item() + + if not math.isfinite(loss_value): + print("Loss is {}, stopping training".format(loss_value)) + print(loss_dict_reduced) + sys.exit(1) + + optimizer.zero_grad() + losses.backward() + optimizer.step() + + if lr_scheduler is not None: + lr_scheduler.step() + + metric_logger.update(loss=losses_reduced, **loss_dict_reduced) + metric_logger.update(lr=optimizer.param_groups[0]["lr"]) + + return metric_logger + + +def _get_iou_types(model): + model_without_ddp = model + if isinstance(model, torch.nn.parallel.DistributedDataParallel): + model_without_ddp = model.module + iou_types = ["bbox"] + if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN): + iou_types.append("segm") + if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN): + iou_types.append("keypoints") + return iou_types + + +@torch.no_grad() +def evaluate(model, data_loader, device): + n_threads = torch.get_num_threads() + # FIXME remove this and make paste_masks_in_image run on the GPU + torch.set_num_threads(1) + cpu_device = torch.device("cpu") + model.eval() + metric_logger = MetricLogger(delimiter=" ") + header = "Test:" + + coco = get_coco_api_from_dataset(data_loader.dataset) + iou_types = _get_iou_types(model) + coco_evaluator = CocoEvaluator(coco, iou_types) + + for images, targets in metric_logger.log_every(data_loader, 100, header): + images = list(img.to(device) for img in images) + + torch.cuda.synchronize() + model_time = time.time() + outputs = model(images) + + outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs] + model_time = time.time() - model_time + + res = { + target["image_id"].item(): output + for target, output in zip(targets, outputs) + } + evaluator_time = time.time() + coco_evaluator.update(res) + evaluator_time = time.time() - evaluator_time + metric_logger.update(model_time=model_time, evaluator_time=evaluator_time) + + # gather the stats from all processes + metric_logger.synchronize_between_processes() + print("Averaged stats:", metric_logger) + coco_evaluator.synchronize_between_processes() + + # accumulate predictions from all images + coco_evaluator.accumulate() + coco_evaluator.summarize() + torch.set_num_threads(n_threads) + return coco_evaluator + + +def get_transform(train): + transforms = [] + transforms.append(ToTensor()) + if train: + transforms.append(RandomHorizontalFlip(0.5)) + return Compose(transforms) + + +def train(): + # train on the GPU or on the CPU, if a GPU is not available + device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") + + # use our dataset and defined transformations + dataset = PennFudanDataset(get_data_path(), get_transform(train=True)) + dataset_test = PennFudanDataset(get_data_path(), get_transform(train=False)) + + # split the dataset in train and test set + indices = torch.randperm(len(dataset)).tolist() + dataset = torch.utils.data.Subset(dataset, indices[:-50]) + dataset_test = torch.utils.data.Subset(dataset_test, indices[-50:]) + + # define training and validation data loaders + data_loader = torch.utils.data.DataLoader( + dataset, batch_size=2, shuffle=True, num_workers=4, collate_fn=collate_fn + ) + + data_loader_test = torch.utils.data.DataLoader( + dataset_test, + batch_size=1, + shuffle=False, + num_workers=4, + collate_fn=collate_fn, + ) + # our dataset has two classes only - background and person + model = pretrained_model( + 2, + weights=MaskRCNN_ResNet50_FPN_Weights.COCO_V1, + weights_backbone=ResNet50_Weights.IMAGENET1K_V1, + ) + model.to(device) + + # construct an optimizer + params = [p for p in model.parameters() if p.requires_grad] + optimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005) + # and a learning rate scheduler + lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.1) + + num_epochs = 10 + for epoch in range(num_epochs): + print(f"-->start to train epoch {epoch}...") + train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq=10) + lr_scheduler.step() + evaluate(model, data_loader_test, device=device) + + _path = get_model_path() + torch.save(model.state_dict(), _path) + print(f"finish model training @ {_path}") + + +if __name__ == "__main__": + train() diff --git a/example/PennFudanPed/code/utils.py b/example/PennFudanPed/pfp/utils/__init__.py similarity index 83% rename from example/PennFudanPed/code/utils.py rename to example/PennFudanPed/pfp/utils/__init__.py index 209d1014c1..fcf8d5e120 100644 --- a/example/PennFudanPed/code/utils.py +++ b/example/PennFudanPed/pfp/utils/__init__.py @@ -1,6 +1,5 @@ import os import time -import errno import pickle import datetime from collections import deque, defaultdict @@ -9,6 +8,22 @@ import torch.distributed as dist +def get_root_path(): + return os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + + +def get_data_path(): + return os.path.join(get_root_path(), "data", "PennFudanPed") + + +def get_model_path(): + return os.path.join(get_root_path(), "models", "mcrnn.pth") + + +def collate_fn(batch): + return tuple(zip(*batch)) + + class SmoothedValue(object): """Track a series of values and provide access to smoothed values over a window or the global series average. @@ -252,10 +267,6 @@ def log_every(self, iterable, print_freq, header=None): ) -def collate_fn(batch): - return tuple(zip(*batch)) - - def warmup_lr_scheduler(optimizer, warmup_iters, warmup_factor): def f(x): if x >= warmup_iters: @@ -266,30 +277,6 @@ def f(x): return torch.optim.lr_scheduler.LambdaLR(optimizer, f) -def mkdir(path): - try: - os.makedirs(path) - except OSError as e: - if e.errno != errno.EEXIST: - raise - - -def setup_for_distributed(is_master): - """ - This function disables printing when not in master process - """ - import builtins as __builtin__ - - builtin_print = __builtin__.print - - def print(*args, **kwargs): - force = kwargs.pop("force", False) - if is_master or force: - builtin_print(*args, **kwargs) - - __builtin__.print = print - - def is_dist_avail_and_initialized(): if not dist.is_available(): return False @@ -302,48 +289,3 @@ def get_world_size(): if not is_dist_avail_and_initialized(): return 1 return dist.get_world_size() - - -def get_rank(): - if not is_dist_avail_and_initialized(): - return 0 - return dist.get_rank() - - -def is_main_process(): - return get_rank() == 0 - - -def save_on_master(*args, **kwargs): - if is_main_process(): - torch.save(*args, **kwargs) - - -def init_distributed_mode(args): - if "RANK" in os.environ and "WORLD_SIZE" in os.environ: - args.rank = int(os.environ["RANK"]) - args.world_size = int(os.environ["WORLD_SIZE"]) - args.gpu = int(os.environ["LOCAL_RANK"]) - elif "SLURM_PROCID" in os.environ: - args.rank = int(os.environ["SLURM_PROCID"]) - args.gpu = args.rank % torch.cuda.device_count() - else: - print("Not using distributed mode") - args.distributed = False - return - - args.distributed = True - - torch.cuda.set_device(args.gpu) - args.dist_backend = "nccl" - print( - "| distributed init (rank {}): {}".format(args.rank, args.dist_url), flush=True - ) - torch.distributed.init_process_group( - backend=args.dist_backend, - init_method=args.dist_url, - world_size=args.world_size, - rank=args.rank, - ) - torch.distributed.barrier() - setup_for_distributed(args.rank == 0) diff --git a/example/PennFudanPed/code/coco_eval.py b/example/PennFudanPed/pfp/utils/coco_eval.py similarity index 75% rename from example/PennFudanPed/code/coco_eval.py rename to example/PennFudanPed/pfp/utils/coco_eval.py index b31d9fe369..fe6665028c 100644 --- a/example/PennFudanPed/code/coco_eval.py +++ b/example/PennFudanPed/pfp/utils/coco_eval.py @@ -1,20 +1,14 @@ +import copy import json +from collections import defaultdict import numpy as np -import copy import torch import torch._six - -from pycocotools.cocoeval import COCOeval -from pycocotools.coco import COCO import pycocotools.mask as mask_util +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval -from collections import defaultdict - -try: - from . import utils -except ImportError: - import utils class CocoEvaluator(object): def __init__(self, coco_gt, iou_types): @@ -48,7 +42,9 @@ def update(self, predictions): def synchronize_between_processes(self): for iou_type in self.iou_types: self.eval_imgs[iou_type] = np.concatenate(self.eval_imgs[iou_type], 2) - create_common_coco_eval(self.coco_eval[iou_type], self.img_ids, self.eval_imgs[iou_type]) + create_common_coco_eval( + self.coco_eval[iou_type], self.img_ids, self.eval_imgs[iou_type] + ) def accumulate(self): for coco_eval in self.coco_eval.values(): @@ -109,7 +105,9 @@ def prepare_for_coco_segmentation(self, predictions): labels = prediction["labels"].tolist() rles = [ - mask_util.encode(np.array(mask[0, :, :, np.newaxis], dtype=np.uint8, order="F"))[0] + mask_util.encode( + np.array(mask[0, :, :, np.newaxis], dtype=np.uint8, order="F") + )[0] for mask in masks ] for rle in rles: @@ -146,7 +144,7 @@ def prepare_for_coco_keypoint(self, predictions): { "image_id": original_id, "category_id": labels[k], - 'keypoints': keypoint, + "keypoints": keypoint, "score": scores[k], } for k, keypoint in enumerate(keypoints) @@ -161,8 +159,10 @@ def convert_to_xywh(boxes): def merge(img_ids, eval_imgs): - all_img_ids = utils.all_gather(img_ids) - all_eval_imgs = utils.all_gather(eval_imgs) + from . import all_gather + + all_img_ids = all_gather(img_ids) + all_eval_imgs = all_gather(eval_imgs) merged_img_ids = [] for p in all_img_ids: @@ -200,27 +200,28 @@ def create_common_coco_eval(coco_eval, img_ids, eval_imgs): # Ideally, pycocotools wouldn't have hard-coded prints # so that we could avoid copy-pasting those two functions + def createIndex(self): # create index # print('creating index...') anns, cats, imgs = {}, {}, {} imgToAnns, catToImgs = defaultdict(list), defaultdict(list) - if 'annotations' in self.dataset: - for ann in self.dataset['annotations']: - imgToAnns[ann['image_id']].append(ann) - anns[ann['id']] = ann + if "annotations" in self.dataset: + for ann in self.dataset["annotations"]: + imgToAnns[ann["image_id"]].append(ann) + anns[ann["id"]] = ann - if 'images' in self.dataset: - for img in self.dataset['images']: - imgs[img['id']] = img + if "images" in self.dataset: + for img in self.dataset["images"]: + imgs[img["id"]] = img - if 'categories' in self.dataset: - for cat in self.dataset['categories']: - cats[cat['id']] = cat + if "categories" in self.dataset: + for cat in self.dataset["categories"]: + cats[cat["id"]] = cat - if 'annotations' in self.dataset and 'categories' in self.dataset: - for ann in self.dataset['annotations']: - catToImgs[ann['category_id']].append(ann['image_id']) + if "annotations" in self.dataset and "categories" in self.dataset: + for ann in self.dataset["annotations"]: + catToImgs[ann["category_id"]].append(ann["image_id"]) # print('index created!') @@ -242,7 +243,7 @@ def loadRes(self, resFile): :return: res (obj) : result api object """ res = COCO() - res.dataset['images'] = [img for img in self.dataset['images']] + res.dataset["images"] = [img for img in self.dataset["images"]] # print('Loading and preparing results...') # tic = time.time() @@ -252,63 +253,70 @@ def loadRes(self, resFile): anns = self.loadNumpyAnnotations(resFile) else: anns = resFile - assert type(anns) == list, 'results in not an array of objects' - annsImgIds = [ann['image_id'] for ann in anns] - assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \ - 'Results do not correspond to current coco set' - if 'caption' in anns[0]: - imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns]) - res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds] + assert type(anns) == list, "results in not an array of objects" + annsImgIds = [ann["image_id"] for ann in anns] + assert set(annsImgIds) == ( + set(annsImgIds) & set(self.getImgIds()) + ), "Results do not correspond to current coco set" + if "caption" in anns[0]: + imgIds = set([img["id"] for img in res.dataset["images"]]) & set( + [ann["image_id"] for ann in anns] + ) + res.dataset["images"] = [ + img for img in res.dataset["images"] if img["id"] in imgIds + ] for id, ann in enumerate(anns): - ann['id'] = id + 1 - elif 'bbox' in anns[0] and not anns[0]['bbox'] == []: - res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + ann["id"] = id + 1 + elif "bbox" in anns[0] and not anns[0]["bbox"] == []: + res.dataset["categories"] = copy.deepcopy(self.dataset["categories"]) for id, ann in enumerate(anns): - bb = ann['bbox'] + bb = ann["bbox"] x1, x2, y1, y2 = [bb[0], bb[0] + bb[2], bb[1], bb[1] + bb[3]] - if 'segmentation' not in ann: - ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]] - ann['area'] = bb[2] * bb[3] - ann['id'] = id + 1 - ann['iscrowd'] = 0 - elif 'segmentation' in anns[0]: - res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + if "segmentation" not in ann: + ann["segmentation"] = [[x1, y1, x1, y2, x2, y2, x2, y1]] + ann["area"] = bb[2] * bb[3] + ann["id"] = id + 1 + ann["iscrowd"] = 0 + elif "segmentation" in anns[0]: + res.dataset["categories"] = copy.deepcopy(self.dataset["categories"]) for id, ann in enumerate(anns): # now only support compressed RLE format as segmentation results - ann['area'] = maskUtils.area(ann['segmentation']) - if 'bbox' not in ann: - ann['bbox'] = maskUtils.toBbox(ann['segmentation']) - ann['id'] = id + 1 - ann['iscrowd'] = 0 - elif 'keypoints' in anns[0]: - res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + ann["area"] = maskUtils.area(ann["segmentation"]) + if "bbox" not in ann: + ann["bbox"] = maskUtils.toBbox(ann["segmentation"]) + ann["id"] = id + 1 + ann["iscrowd"] = 0 + elif "keypoints" in anns[0]: + res.dataset["categories"] = copy.deepcopy(self.dataset["categories"]) for id, ann in enumerate(anns): - s = ann['keypoints'] + s = ann["keypoints"] x = s[0::3] y = s[1::3] x1, x2, y1, y2 = np.min(x), np.max(x), np.min(y), np.max(y) - ann['area'] = (x2 - x1) * (y2 - y1) - ann['id'] = id + 1 - ann['bbox'] = [x1, y1, x2 - x1, y2 - y1] + ann["area"] = (x2 - x1) * (y2 - y1) + ann["id"] = id + 1 + ann["bbox"] = [x1, y1, x2 - x1, y2 - y1] # print('DONE (t={:0.2f}s)'.format(time.time()- tic)) - res.dataset['annotations'] = anns + res.dataset["annotations"] = anns createIndex(res) return res def evaluate(self): - ''' + """ Run per image evaluation on given images and store results (a list of dict) in self.evalImgs :return: None - ''' + """ # tic = time.time() # print('Running per image evaluation...') p = self.params # add backward compatibility if useSegm is specified in params if p.useSegm is not None: - p.iouType = 'segm' if p.useSegm == 1 else 'bbox' - print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType)) + p.iouType = "segm" if p.useSegm == 1 else "bbox" + print( + "useSegm (deprecated) is not None. Running {} evaluation".format(p.iouType) + ) # print('Evaluate annotation type *{}*'.format(p.iouType)) p.imgIds = list(np.unique(p.imgIds)) if p.useCats: @@ -320,14 +328,15 @@ def evaluate(self): # loop through images, area range, max detection number catIds = p.catIds if p.useCats else [-1] - if p.iouType == 'segm' or p.iouType == 'bbox': + if p.iouType == "segm" or p.iouType == "bbox": computeIoU = self.computeIoU - elif p.iouType == 'keypoints': + elif p.iouType == "keypoints": computeIoU = self.computeOks self.ious = { (imgId, catId): computeIoU(imgId, catId) for imgId in p.imgIds - for catId in catIds} + for catId in catIds + } evaluateImg = self.evaluateImg maxDet = p.maxDets[-1] @@ -344,6 +353,7 @@ def evaluate(self): # print('DONE (t={:0.2f}s).'.format(toc-tic)) return p.imgIds, evalImgs + ################################################################# # end of straight copy from pycocotools, just removing the prints ################################################################# diff --git a/example/PennFudanPed/pfp/utils/coco_utils.py b/example/PennFudanPed/pfp/utils/coco_utils.py new file mode 100644 index 0000000000..76d4f2ae2e --- /dev/null +++ b/example/PennFudanPed/pfp/utils/coco_utils.py @@ -0,0 +1,93 @@ +import torch +import torchvision +import torch.utils.data +from pycocotools import mask as coco_mask +from pycocotools.coco import COCO + + +def convert_to_coco_api(ds): + coco_ds = COCO() + # annotation IDs need to start at 1, not 0, see torchvision issue #1530 + ann_id = 1 + dataset = {"images": [], "categories": [], "annotations": []} + categories = set() + img_idx = 0 + for img, targets in ds: + # find better way to get target + # targets = ds.get_annotations(img_idx) + # img, targets = ds[img_idx] + image_id = targets["image_id"].item() + img_dict = {} + img_dict["id"] = image_id + if isinstance(img, torch.Tensor): + img_dict["height"] = img.shape[-2] + img_dict["width"] = img.shape[-1] + else: + img_dict["height"] = img["height"] + img_dict["width"] = img["width"] + dataset["images"].append(img_dict) + + bboxes = targets["boxes"] + bboxes[:, 2:] -= bboxes[:, :2] + bboxes = bboxes.tolist() + + labels = targets["labels"].tolist() + areas = targets["area"].tolist() + iscrowd = targets["iscrowd"].tolist() + + if "masks" in targets: + masks = targets["masks"] + # make masks Fortran contiguous for coco_mask + masks = masks.permute(0, 2, 1).contiguous().permute(0, 2, 1) + + if "keypoints" in targets: + keypoints = targets["keypoints"] + keypoints = keypoints.reshape(keypoints.shape[0], -1).tolist() + + num_objs = len(bboxes) + for i in range(num_objs): + ann = {} + ann["image_id"] = image_id + ann["bbox"] = bboxes[i] + ann["category_id"] = labels[i] + categories.add(labels[i]) + ann["area"] = areas[i] + ann["iscrowd"] = iscrowd[i] + ann["id"] = ann_id + if "masks" in targets: + ann["segmentation"] = coco_mask.encode(masks[i].numpy()) + if "keypoints" in targets: + ann["keypoints"] = keypoints[i] + ann["num_keypoints"] = sum(k != 0 for k in keypoints[i][2::3]) + dataset["annotations"].append(ann) + ann_id += 1 + img_idx += 1 + dataset["categories"] = [{"id": i} for i in sorted(categories)] + coco_ds.dataset = dataset + coco_ds.createIndex() + return coco_ds + + +def get_coco_api_from_dataset(dataset): + for _ in range(10): + if isinstance(dataset, torchvision.datasets.CocoDetection): + break + if isinstance(dataset, torch.utils.data.Subset): + dataset = dataset.dataset + if isinstance(dataset, torchvision.datasets.CocoDetection): + return dataset.coco + return convert_to_coco_api(dataset) + + +class CocoDetection(torchvision.datasets.CocoDetection): + def __init__(self, img_folder, ann_file, transforms): + super(CocoDetection, self).__init__(img_folder, ann_file) + self._transforms = transforms + + def __getitem__(self, idx): + img, target = super(CocoDetection, self).__getitem__(idx) + image_id = self.ids[idx] + target = dict(image_id=image_id, annotations=target) + if self._transforms is not None: + img, target = self._transforms(img, target) + return img, target diff --git a/example/PennFudanPed/requirements.txt b/example/PennFudanPed/requirements.txt deleted file mode 100644 index 747fe24885..0000000000 --- a/example/PennFudanPed/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -pycocotools -numpy -torch -torchvision diff --git a/example/PennFudanPed/runtime.yaml b/example/PennFudanPed/runtime.yaml deleted file mode 100644 index 08c535eb61..0000000000 --- a/example/PennFudanPed/runtime.yaml +++ /dev/null @@ -1,5 +0,0 @@ -mode: venv -name: vision_pytorch -pip_req: requirements.txt -python_version: '3.8' -starwhale_version: 0.2.0b8 diff --git a/example/runtime/pytorch/requirements-sw-lock.txt b/example/runtime/pytorch/requirements-sw-lock.txt index dc1b642fff..fbd6aa1e81 100644 --- a/example/runtime/pytorch/requirements-sw-lock.txt +++ b/example/runtime/pytorch/requirements-sw-lock.txt @@ -1,7 +1,7 @@ -# Generated by Starwhale(0.0.0.dev0) Runtime Lock ---index-url 'https://pypi.doubanio.com/simple/' ---extra-index-url 'https://mirrors.bfsu.edu.cn/pypi/web/simple/' ---trusted-host 'mirrors.bfsu.edu.cn\npypi.doubanio.com' +# Generated by Starwhale(0.3.0rc2) Runtime Lock +--index-url 'https://pypi.tuna.tsinghua.edu.cn/simple' +--extra-index-url 'https://pypi.doubanio.com/simple' +--trusted-host 'pypi.tuna.tsinghua.edu.cn pypi.doubanio.com' appdirs==1.4.4 attrs==21.4.0 boto3==1.21.0 @@ -12,32 +12,52 @@ charset-normalizer==2.1.0 click==8.1.3 commonmark==0.9.1 conda-pack==0.6.0 +cycler==0.11.0 dill==0.3.5.1 distlib==0.3.5 +dummy @ file:///home/liutianwei/.cache/starwhale/self/workdir/runtime/pytorch/gz/gzsdsnjsmq4wkzjyg44tkzjwof3xm2q/wheels/dummy-0.0.0-py3-none-any.whl filelock==3.7.1 +fonttools==4.37.1 fs==2.4.16 idna==3.3 importlib-metadata==4.12.0 +Jinja2==3.1.2 jmespath==0.10.0 joblib==1.1.0 jsonlines==3.0.0 +kiwisolver==1.4.4 loguru==0.6.0 -numpy==1.23.1 +MarkupSafe==2.1.1 +matplotlib==3.5.3 +numpy==1.23.2 +packaging==21.3 Pillow==9.2.0 platformdirs==2.5.2 +portalocker==2.5.1 +pyarrow==9.0.0 +pycocotools==2.0.4 Pygments==2.12.0 +pyparsing==3.0.9 python-dateutil==2.8.2 PyYAML==6.0 requests==2.28.1 requests-toolbelt==0.9.1 -rich==12.0.0 +rich==12.5.1 s3transfer==0.5.2 scikit-learn==1.1.1 scipy==1.8.1 +shellingham==1.5.0 six==1.16.0 +starwhale==0.3.0rc2 +tenacity==8.0.1 +textual==0.1.18 threadpoolctl==3.1.0 -torch==1.12.0 -torchvision==0.13.0 +torch==1.12.1 +torchaudio==0.12.1 +torchdata==0.4.1 +torchtext==0.13.1 +torchvision==0.13.1 +tqdm==4.64.0 typing_extensions==4.3.0 urllib3==1.26.10 virtualenv==20.15.1 diff --git a/example/runtime/pytorch/runtime.yaml b/example/runtime/pytorch/runtime.yaml index 36a89041fa..0d6ccedcec 100644 --- a/example/runtime/pytorch/runtime.yaml +++ b/example/runtime/pytorch/runtime.yaml @@ -6,15 +6,22 @@ configs: docker: image: ghcr.io/star-whale/runtime/pytorch pip: - extra_index_url: https://mirrors.bfsu.edu.cn/pypi/web/simple/ - index_url: https://pypi.doubanio.com/simple/ + extra_index_url: https://pypi.doubanio.com/simple + index_url: https://pypi.tuna.tsinghua.edu.cn/simple trusted_host: - - mirrors.bfsu.edu.cn + - pypi.tuna.tsinghua.edu.cn - pypi.doubanio.com dependencies: - pip: - - numpy >= 1.23.2 + - Pillow + - numpy - scikit-learn + - torchvision + - torch + - torchdata + - torchtext + - torchaudio + - pycocotools - wheels: - dummy-0.0.0-py3-none-any.whl - files: