Skip to content
This repository has been archived by the owner on Mar 19, 2023. It is now read-only.

Commit

Permalink
Merge pull request #37 from robmarkcole/adds-confidence
Browse files Browse the repository at this point in the history 
Adds confidence
  • Loading branch information
@robmarkcole
robmarkcole authored Jul 16, 2019
2 parents 23e5e2d + 4f4a65c commit f8b969d
Show file tree
Hide file tree
Showing 4 changed files with 194 additions and 43 deletions.
4 changes: 3 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ Deepstack [object detection](https://deepstackpython.readthedocs.io/en/latest/ob
sudo docker run -e VISION-DETECTION=True -v localstorage:/datastore -p 5000:5000 deepquestai/deepstack
```

The `deepstack_object` component adds an `image_processing` entity where the state of the entity is the total number of `target` objects that are found in the camera image. The class and number objects of each class is listed in the entity attributes. An event `image_processing.object_detected` is fired for each object detected. Optionally the processed image can be saved to disk. If `save_file_folder` is configured two images are created, one with the filename of format `deepstack_latest_{target}.jpg` which is over-written on each new detection of the `target`, and another with a unique filename including the timestamp.
The `deepstack_object` component adds an `image_processing` entity where the state of the entity is the total number of `target` objects that are above a `confidence` threshold which has a default value of 80%. The class and number of objects of each object detected (any confidence) is listed in the entity attributes. An event `image_processing.object_detected` is fired for each object detected. Optionally the processed image can be saved to disk. If `save_file_folder` is configured two images are created, one with the filename of format `deepstack_latest_{target}.jpg` which is over-written on each new detection of the `target`, and another with a unique filename including the timestamp.

Add to your Home-Assistant config:
```yaml
Expand All @@ -40,6 +40,7 @@ image_processing:
scan_interval: 20000
save_file_folder: /config/www/deepstack_person_images
target: person
confidence: 50
source:
- entity_id: camera.local_file
name: person_detector
Expand All @@ -50,6 +51,7 @@ Configuration variables:
- **save_file_folder**: (Optional) The folder to save processed images to. Note that folder path should be added to [whitelist_external_dirs](https://www.home-assistant.io/docs/configuration/basic/)
- **source**: Must be a camera.
- **target**: The target object class, default `person`.
- **confidence**: (Optional) The confidence (in %) above which detected targets are counted in the sensor state. Default value: 80
- **name**: (Optional) A custom name for the the entity.

<p align="center">
Expand Down
62 changes: 43 additions & 19 deletions custom_components/deepstack_object/image_processing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from homeassistant.core import split_entity_id
import homeassistant.helpers.config_validation as cv
from homeassistant.components.image_processing import (
PLATFORM_SCHEMA, ImageProcessingEntity, CONF_SOURCE,
PLATFORM_SCHEMA, ImageProcessingEntity, ATTR_CONFIDENCE, CONF_SOURCE,
CONF_ENTITY_ID, CONF_NAME, DOMAIN)
from homeassistant.const import (
CONF_IP_ADDRESS, CONF_PORT,
Expand All @@ -28,7 +28,6 @@
CLASSIFIER = 'deepstack_object'
CONF_TARGET = 'target'
CONF_SAVE_FILE_FOLDER = 'save_file_folder'
CONFIDENCE = 'confidence'
DEFAULT_TARGET = 'person'
EVENT_OBJECT_DETECTED = 'image_processing.object_detected'
EVENT_FILE_SAVED = 'image_processing.file_saved'
Expand All @@ -44,12 +43,26 @@
})


def draw_box(draw, prediction, color=(255, 0, 0)):
def draw_box(draw, prediction, text='', color=(255, 0, 0)):
"""Draw bounding box on image."""
(left, right, top, bottom) = (
prediction['x_min'], prediction['x_max'], prediction['y_min'], prediction['y_max'])
draw.line([(left, top), (left, bottom), (right, bottom),
(right, top), (left, top)], width=5, fill=color)
if text:
draw.text((left, abs(top-15)), text, fill=color)


def format_confidence(confidence):
"""Takes a confidence from the API like
0.55623 and returne 55.6 (%).
"""
return round(float(confidence)*100, 1)


def get_confidences_above_threshold(confidences, threshold):
"""Takes a list of confidences and returns those above a threshold."""
return [val for val in confidences if val >= threshold]


def get_object_classes(predictions):
Expand All @@ -64,17 +77,17 @@ def get_object_instances(predictions, target):
"""
Return the number of instances of a target class.
"""
targets_identified = [
pred for pred in predictions if pred['label'] == target]
return len(targets_identified)
targets_identified = [format_confidence(
pred['confidence']) for pred in predictions if pred['label'] == target]
return targets_identified


def get_objects_summary(predictions):
"""
Get a summary of the objects detected.
"""
classes = get_object_classes(predictions)
return {class_cat: get_object_instances(predictions, target=class_cat)
return {class_cat: len(get_object_instances(predictions, target=class_cat))
for class_cat in classes}


Expand All @@ -97,6 +110,7 @@ def setup_platform(hass, config, add_devices, discovery_info=None):
port = config.get(CONF_PORT)
target = config.get(CONF_TARGET)
save_file_folder = config.get(CONF_SAVE_FILE_FOLDER)
confidence = config.get(ATTR_CONFIDENCE)

if save_file_folder:
save_file_folder = os.path.join(
Expand All @@ -105,7 +119,7 @@ def setup_platform(hass, config, add_devices, discovery_info=None):
entities = []
for camera in config[CONF_SOURCE]:
object_entity = ObjectClassifyEntity(
ip_address, port, target, save_file_folder,
ip_address, port, target, confidence, save_file_folder,
camera[CONF_ENTITY_ID], camera.get(CONF_NAME))
entities.append(object_entity)
add_devices(entities)
Expand All @@ -114,19 +128,21 @@ def setup_platform(hass, config, add_devices, discovery_info=None):
class ObjectClassifyEntity(ImageProcessingEntity):
"""Perform a face classification."""

def __init__(self, ip_address, port, target, save_file_folder, camera_entity, name=None):
def __init__(self, ip_address, port, target, confidence, save_file_folder, camera_entity, name=None):
"""Init with the API key and model id."""
super().__init__()
self._url_check = "http://{}:{}/v1/vision/detection".format(
ip_address, port)
self._target = target
self._confidence = confidence
self._camera = camera_entity
if name:
self._name = name
else:
camera_name = split_entity_id(camera_entity)[1]
self._name = "{} {}".format(CLASSIFIER, camera_name)
self._state = None
self._targets_confidences = []
self._predictions = {}
if save_file_folder:
self._save_file_folder = save_file_folder
Expand All @@ -139,8 +155,11 @@ def process_image(self, image):
if response:
if response.status_code == HTTP_OK:
predictions_json = response.json()["predictions"]
self._state = get_object_instances(
self._targets_confidences = get_object_instances(
predictions_json, self._target)
self._state = len(
get_confidences_above_threshold(
self._targets_confidences, self._confidence))
self._predictions = get_objects_summary(predictions_json)
self.fire_prediction_events(predictions_json)
if hasattr(self, "_save_file_folder") and self._state > 0:
Expand All @@ -149,6 +168,7 @@ def process_image(self, image):

else:
self._state = None
self._targets_confidences = []
self._predictions = {}

def save_image(self, image, predictions_json, target, directory):
Expand All @@ -159,16 +179,18 @@ def save_image(self, image, predictions_json, target, directory):
draw = ImageDraw.Draw(img)

for prediction in predictions_json:
if prediction['label'] == target:
draw_box(draw, prediction)
prediction_confidence = format_confidence(prediction['confidence'])
if prediction['label'] == target and prediction_confidence >= self._confidence:
draw_box(draw, prediction, str(prediction_confidence))

now = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
save_path = directory + 'deepstack_{}_{}.jpg'.format(target, now)
latest_save_path = directory + 'deepstack_latest_{}.jpg'.format(target)
timestamp_save_path = directory + 'deepstack_{}_{}.jpg'.format(target, now)
try:
img.save(directory + 'deepstack_latest_{}.jpg'.format(target))
img.save(save_path)
self.fire_saved_file_event(save_path)
_LOGGER.info("Saved bounding box image to %s", save_path)
img.save(latest_save_path)
img.save(timestamp_save_path)
self.fire_saved_file_event(timestamp_save_path)
_LOGGER.info("Saved bounding box image to %s", timestamp_save_path)
except Exception as exc:
_LOGGER.error("Error saving bounding box image : %s", exc)

Expand All @@ -181,7 +203,8 @@ def fire_prediction_events(self, predictions_json):
'classifier': CLASSIFIER,
ATTR_ENTITY_ID: self.entity_id,
OBJECT: prediction['label'],
CONFIDENCE: prediction['confidence'],
ATTR_CONFIDENCE: format_confidence(
prediction['confidence'])
})

def fire_saved_file_event(self, save_path):
Expand Down Expand Up @@ -213,7 +236,8 @@ def device_state_attributes(self):
"""Return device specific state attributes."""
attr = {}
attr['target'] = self._target
attr['predictions'] = self._predictions
attr['target_confidences'] = self._targets_confidences
attr['all_predictions'] = self._predictions
if hasattr(self, "_save_file_folder"):
attr['save_file_folder'] = self._save_file_folder
return attr
Expand Down
68 changes: 60 additions & 8 deletions development/.ipynb_checkpoints/Deepstack object detection-checkpoint.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,30 +223,80 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 23,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"55.6"
]
},
"execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def format_confidece(confidence):\n",
" \"\"\"Takes a confidence from the API like \n",
" 0.55623 and returne 55.6 (%).\n",
" \"\"\"\n",
" return round(confidence*100, 1)\n",
"\n",
"format_confidece(0.55623)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[0.5, 0.6]"
]
},
"execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"def get_confidences_above_threshold(confidences, threshold):\n",
" \"\"\"Takes a list of confidences and returns those above a threshold.\"\"\"\n",
" return [val for val in confidences if val >= threshold]\n",
"\n",
"get_confidences_above_threshold([0.2, 0.5, 0.6], 0.5)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"def get_object_instances(predictions, target):\n",
" \"\"\"\n",
" Return the number of instances of a target class.\n",
" \"\"\"\n",
" targets_identified = [pred for pred in predictions if pred['label'] == target]\n",
" return len(targets_identified)"
" targets_identified = [format_confidece(pred['confidence']) for pred in predictions if pred['label'] == target]\n",
" return targets_identified"
]
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 27,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"2"
"[100.0, 99.9]"
]
},
"execution_count": 13,
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -303,10 +353,12 @@
"metadata": {},
"outputs": [],
"source": [
"def draw_box(draw, prediction, color=(255, 0, 0)):\n",
"def draw_box(draw, prediction, text='', color=(255, 0, 0)):\n",
" \"\"\"Draw bounding box on image.\"\"\"\n",
" (left, right, top, bottom) = (prediction['x_min'], prediction['x_max'], prediction['y_min'], prediction['y_max'])\n",
" draw.line([(left, top), (left, bottom), (right, bottom), (right, top), (left, top)], width=5, fill=color)"
" draw.line([(left, top), (left, bottom), (right, bottom), (right, top), (left, top)], width=5, fill=color)\n",
" if text:\n",
" draw.text((left, abs(top-15)), text, fill=color)"
]
},
{
Expand Down
Loading

0 comments on commit f8b969d

Please sign in to comment.