From ded158abd1bb9d0a0cb9848281fad1d4fddc8cca Mon Sep 17 00:00:00 2001 From: Omkar Prabhu Date: Sat, 9 Mar 2024 14:50:16 +0530 Subject: [PATCH] Getting ready for next release with more updates (#62) * Getting ready for next release with more updates * Fixed linter errors * Updated codecov --- api/config/config.go | 4 +- api/internal/handlers/handlers.go | 7 +++ codecov.yml | 2 +- docker-compose.yaml | 51 +++++++++--------- docs/docs/dev-guide/roadmap.md | 4 +- docs/docs/user-guide/features/search.md | 2 +- docs/docs/user-guide/features/things.md | 2 +- docs/package-lock.json | 4 +- docs/package.json | 2 +- docs/swagger.yaml | 10 +++- scripts/generate_test_data.py | 43 ++++++++------- worker/requirements.txt | 2 +- worker/src/components/metadata.py | 69 +++++++++++++++++-------- worker/src/providers/search/pytorch.py | 4 +- 14 files changed, 127 insertions(+), 79 deletions(-) diff --git a/api/config/config.go b/api/config/config.go index 681ce9f..402deed 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -63,11 +63,11 @@ type ( Faces bool `envconfig:"SMRITI_ML_FACES" default:"true"` PlacesProvider string `envconfig:"SMRITI_ML_PLACES_PROVIDER" default:"openstreetmap"` ClassificationProvider string `envconfig:"SMRITI_ML_CLASSIFICATION_PROVIDER" default:"pytorch"` - ClassificationParams string `envconfig:"SMRITI_ML_CLASSIFICATION_PARAMS" default:"{\"file\":\"classification_v240229.pt\"}"` + ClassificationParams string `envconfig:"SMRITI_ML_CLASSIFICATION_PARAMS" default:"{\"file\":\"classification_v240309.pt\"}"` OCRProvider string `envconfig:"SMRITI_ML_OCR_PROVIDER" default:"paddlepaddle"` OCRParams string `envconfig:"SMRITI_ML_OCR_PARAMS" default:"{\"det_model_dir\":\"det_onnx\",\"rec_model_dir\":\"rec_onnx\",\"cls_model_dir\":\"cls_onnx\"}"` SearchProvider string `envconfig:"SMRITI_ML_SEARCH_PROVIDER" default:"pytorch"` - SearchParams string `envconfig:"SMRITI_ML_SEARCH_PARAMS" default:"{\"tokenizer_dir\":\"search_tokenizer\",\"processor_dir\":\"search_processor\",\"text_file\":\"search_text_v240229.pt\",\"vision_file\":\"search_vision_v240229.pt\"}"` //nolint:lll + SearchParams string `envconfig:"SMRITI_ML_SEARCH_PARAMS" default:"{\"tokenizer_dir\":\"search_tokenizer\",\"processor_dir\":\"search_processor\",\"text_file\":\"search_text_v240309.pt\",\"vision_file\":\"search_vision_v240309.pt\"}"` //nolint:lll FacesProvider string `envconfig:"SMRITI_ML_FACES_PROVIDER" default:"pytorch"` FacesParams string `envconfig:"SMRITI_ML_FACES_PARAMS" default:"{\"minutes\":\"1\",\"face_threshold\":\"0.9\",\"model\":\"vggface2\",\"clustering\":\"annoy\"}"` MetadataParams string `envconfig:"SMRITI_ML_METADATA_PARAMS" default:"{\"thumbnail_size\":\"512\"}"` diff --git a/api/internal/handlers/handlers.go b/api/internal/handlers/handlers.go index 0009a2c..24e7306 100644 --- a/api/internal/handlers/handlers.go +++ b/api/internal/handlers/handlers.go @@ -2,6 +2,7 @@ package handlers import ( "api/config" + "api/internal/models" "api/pkg/cache" "api/pkg/services/worker" "fmt" @@ -68,6 +69,12 @@ func getMediaItemFilters(ctx echo.Context) string { if mediaItemCategory != "" { filterQuery += fmt.Sprintf(" AND mediaitem_category = '%s'", mediaItemCategory) } + mediaItemStatus := ctx.QueryParam("status") + if mediaItemStatus != "" && (mediaItemStatus == string(models.Unspecified) || + mediaItemStatus == string(models.Ready) || mediaItemStatus == string(models.Processing) || + mediaItemStatus == string(models.Failed)) { + filterQuery += fmt.Sprintf(" AND status = '%s'", mediaItemStatus) + } return filterQuery } diff --git a/codecov.yml b/codecov.yml index 5d5f44d..a5b081f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -2,7 +2,7 @@ coverage: status: project: default: - target: 90% + target: 60% threshold: 10% paths: - api diff --git a/docker-compose.yaml b/docker-compose.yaml index b87dd58..1e0d03c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -21,11 +21,11 @@ services: SMRITI_STORAGE_SECRET_KEY: smritipass volumes: - ./storage:/storage:rw - deploy: - resources: - limits: - cpus: '0.2' - memory: 256M + # deploy: + # resources: + # limits: + # cpus: '0.2' + # memory: 256M worker: container_name: worker build: worker @@ -39,14 +39,15 @@ services: SMRITI_API_HOST: api PYTHONUNBUFFERED: '1' TOKENIZERS_PARALLELISM: false + SMRITI_LOG_LEVEL: DEBUG volumes: - ./storage:/storage:rw - ./models:/models:rw - deploy: - resources: - limits: - cpus: '1.2' - memory: 3G + # deploy: + # resources: + # limits: + # cpus: '1.2' + # memory: 3G # infra services database: container_name: database @@ -58,11 +59,11 @@ services: POSTGRES_DB: smriti ports: - '5432:5432' - deploy: - resources: - limits: - cpus: '0.2' - memory: 256M + # deploy: + # resources: + # limits: + # cpus: '0.2' + # memory: 256M cache: container_name: cache image: redis:7.2.4-alpine @@ -71,11 +72,11 @@ services: --requirepass smritipass ports: - '6379:6379' - deploy: - resources: - limits: - cpus: '0.2' - memory: 256M + # deploy: + # resources: + # limits: + # cpus: '0.2' + # memory: 256M storage: container_name: storage image: minio/minio:RELEASE.2024-02-04T22-36-13Z @@ -88,8 +89,8 @@ services: MINIO_ROOT_PASSWORD: smritipass command: server --address 0.0.0.0:9000 /storage restart: always - deploy: - resources: - limits: - cpus: '0.2' - memory: 256M \ No newline at end of file + # deploy: + # resources: + # limits: + # cpus: '0.2' + # memory: 256M \ No newline at end of file diff --git a/docs/docs/dev-guide/roadmap.md b/docs/docs/dev-guide/roadmap.md index fdff31f..d96a961 100644 --- a/docs/docs/dev-guide/roadmap.md +++ b/docs/docs/dev-guide/roadmap.md @@ -13,7 +13,7 @@ - [x] Get Machine Learning Powered Faces & People Collection (v2023.08.31) - [x] Planning recurrent upgrades of third party libraries (v2023.08.31) - [x] Open up for contributions to build Mobile & Web Application (v2023.08.31) -- [ ] Enhance Image & Video Support (v2024.02.29) -- [ ] Improvements to E2E Test Suite (v2024.02.29) +- [x] Enhance Image & Video Support (v2024.03.09) +- [x] Improvements to E2E Test Suite (v2024.03.09) - [ ] Sync content from Social Media Websites (v2024.04.30) - [ ] Generation of Memories (v2024.04.30) diff --git a/docs/docs/user-guide/features/search.md b/docs/docs/user-guide/features/search.md index 8b08f74..129e300 100644 --- a/docs/docs/user-guide/features/search.md +++ b/docs/docs/user-guide/features/search.md @@ -3,7 +3,7 @@ Enable this feature using API Configuration: ```bash SMRITI_ML_SEARCH: true SMRITI_ML_SEARCH_PROVIDER: pytorch -SMRITI_ML_SEARCH_PARAMS: {"tokenizer_dir":"search_tokenizer","processor_dir":"search_processor","text_file":"search_text_v240229.pt","vision_file":"search_vision_v240229.pt"} +SMRITI_ML_SEARCH_PARAMS: {"tokenizer_dir":"search_tokenizer","processor_dir":"search_processor","text_file":"search_text_v240309.pt","vision_file":"search_vision_v240309.pt"} ``` ## Use Cases diff --git a/docs/docs/user-guide/features/things.md b/docs/docs/user-guide/features/things.md index eddf2c1..909031f 100644 --- a/docs/docs/user-guide/features/things.md +++ b/docs/docs/user-guide/features/things.md @@ -4,7 +4,7 @@ Enable this feature using API Configuration: SMRITI_FEATURE_THINGS: true SMRITI_ML_CLASSIFICATION: true SMRITI_ML_CLASSIFICATION_PROVIDER: pytorch -SMRITI_ML_CLASSIFICATION_PARAMS: {"file":"classification_v240229.pt"} +SMRITI_ML_CLASSIFICATION_PARAMS: {"file":"classification_v240309.pt"} ``` ## Use Cases diff --git a/docs/package-lock.json b/docs/package-lock.json index 6bdf137..54439e7 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -1,12 +1,12 @@ { "name": "smriti", - "version": "24.02.29", + "version": "24.03.09", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "smriti", - "version": "24.02.29", + "version": "24.03.09", "dependencies": { "@babel/traverse": "^7.23.9", "@docusaurus/core": "^3.1.1", diff --git a/docs/package.json b/docs/package.json index 6fcea8c..93282d2 100644 --- a/docs/package.json +++ b/docs/package.json @@ -1,6 +1,6 @@ { "name": "smriti", - "version": "24.02.29", + "version": "24.03.09", "private": true, "scripts": { "docusaurus": "docusaurus", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9b3e3c5..03efecd 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -2,7 +2,7 @@ openapi: 3.0.3 info: title: Smriti API description: Smarter Home for all your Photos and Videos - version: 24.02.29 + version: 24.03.09 servers: - url: https://localhost:5001 tags: @@ -146,6 +146,12 @@ paths: schema: type: string example: live + - name: status + in: query + description: MediaItem Status + schema: + type: string + example: FAILED summary: Get all existing mediaItems description: Get all existing mediaItems operationId: getMediaItems @@ -1323,7 +1329,7 @@ components: properties: version: type: string - example: 24.02.29 + example: 24.03.09 gitSha: type: string example: addf120b430021c36c232c99ef8d926aea2acd6b diff --git a/scripts/generate_test_data.py b/scripts/generate_test_data.py index 66d857a..48252f3 100644 --- a/scripts/generate_test_data.py +++ b/scripts/generate_test_data.py @@ -4,10 +4,11 @@ import zipfile import shutil import time +import multiprocessing as mp API_URL ='http://localhost:5001' -DOWNLOAD_PHOTOS_URL = 'https://www.dropbox.com/sh/q7yqg7uufaflqjg/AABAo-QEwNIAjxyZSJD0ICzDa?dl=1' +DOWNLOAD_SAMPLES_URL = 'https://www.dropbox.com/scl/fo/yyy82163nqh5ii5aqm8vz/h?rlkey=bjlvrz198fu9zntu6tmvgb2ya&dl=1' # create user res = requests.post(f'{API_URL}/v1/users', auth=('smriti', 'smritiT3st!'), json={'name':'Jeff Dean','username':'jeffdean','password':'jeffT3st!','features':'{"albums":true,'+ @@ -28,11 +29,29 @@ print('✅ got auth token') headers = {'Authorization': f'Bearer {access_token}'} +# upload function +def upload(file_type, file): + files = {'file': open(f'samples/{file_type}/{file}', 'rb')} + res = requests.post(f'{API_URL}/v1/mediaItems', files=files, headers=headers) + if res.status_code != 201: + res = res.json() + print(f'❌ error uploading sample mediaitem: {res["message"]}') + exit(0) + res = res.json() + while True: + res = requests.get(f'{API_URL}/v1/mediaItems/{res["id"]}', files=files, headers=headers) + assert res.status_code == 200 + res = res.json() + if res['status'] == 'READY': + break + time.sleep(5) + return res['id'] + # download and upload mediaitems mediaitems = [] print('ℹ️ downloading sample mediaitems, hang on...') if not os.path.exists('samples.zip'): - response = requests.get(DOWNLOAD_PHOTOS_URL) + response = requests.get(DOWNLOAD_SAMPLES_URL) if response.status_code == 200: local_file_path = 'samples.zip' with open(local_file_path, 'wb') as local_file: @@ -45,22 +64,10 @@ with zipfile.ZipFile('samples.zip', 'r') as zip_ref: zip_ref.extractall('samples') print('ℹ️ uploading sample mediaitems, hang on...') -for file in os.listdir('samples'): - files = {'file': open(f'samples/{file}', 'rb')} - res = requests.post(f'{API_URL}/v1/mediaItems', files=files, headers=headers) - if res.status_code != 201: - res = res.json() - print(f'❌ error uploading sample mediaitem: {res["message"]}') - exit(0) - res = res.json() - mediaitems.append(res['id']) - while True: - res = requests.get(f'{API_URL}/v1/mediaItems/{res["id"]}', files=files, headers=headers) - assert res.status_code == 200 - res = res.json() - if res['status'] == 'READY': - break - time.sleep(2) +for file_type in os.listdir('samples'): + files = os.listdir(f'samples/{file_type}') + with mp.Pool(processes=mp.cpu_count()-1) as pool: + mediaitems = pool.starmap(upload, list(zip([file_type for _ in range(len(files))], files)), chunksize=2) print('✅ uploaded sample mediaitems') # create albums diff --git a/worker/requirements.txt b/worker/requirements.txt index aa48a30..ac85c71 100644 --- a/worker/requirements.txt +++ b/worker/requirements.txt @@ -10,7 +10,7 @@ opencv-python==4.9.0.80 prometheus-client==0.19.0 Pillow==10.2.0 PyExifTool==0.5.6 -rawpy==0.19.0 +rawpy==0.19.1 requests==2.31.0 schedule==1.2.1 torch==2.2.0 diff --git a/worker/src/components/metadata.py b/worker/src/components/metadata.py index 3ab53d9..07afeb3 100644 --- a/worker/src/components/metadata.py +++ b/worker/src/components/metadata.py @@ -55,11 +55,12 @@ async def process(self, mediaitem_user_id: str, mediaitem_id: str, mediaitem_fil result['category'] = 'default' try: with exiftool.ExifToolHelper() as ethelper: + ethelper.check_execute = False metadata = ethelper.get_metadata(file_path)[0] logging.debug(f'metadata for user {mediaitem_user_id} mediaitem {mediaitem_id}: {metadata}') result['mimeType'] = getval_from_dict(metadata, ['File:MIMEType']) - result['type'] = 'photo' if 'image' in metadata['File:MIMEType'] else \ - 'video' if 'video' in metadata['File:MIMEType'] else 'unknown' + result['type'] = 'photo' if result['mimeType'] and 'image' in result['mimeType'] else \ + 'video' if result['mimeType'] and 'video' in result['mimeType'] else 'unknown' result['width'] = getval_from_dict(metadata, ['EXIF:SensorWidth', 'EXIF:ImageWidth', 'EXIF:ExifImageWidth', 'File:ImageWidth', 'PNG:ImageWidth', 'XMP:ExifImageWidth', @@ -70,7 +71,7 @@ async def process(self, mediaitem_user_id: str, mediaitem_id: str, mediaitem_fil 'PNG:ImageHeight', 'XMP:ExifImageHeight', 'QuickTime:ImageHeight', 'QuickTime:SourceImageHeight'], return_type='int') - if result['height'] is None or result['width'] is None and 'Composite:ImageSize' in metadata: + if (result['height'] is None or result['width'] is None) and 'Composite:ImageSize' in metadata: composite_dims = metadata['Composite:ImageSize'].split(' ') if len(composite_dims) == 2: result['width'] = int(composite_dims[0]) @@ -133,7 +134,7 @@ async def process(self, mediaitem_user_id: str, mediaitem_id: str, mediaitem_fil self._grpc_save_mediaitem_metadata(result) return None - if result['type'] == 'photo': + if result['type'] == 'photo' or result['type'] == 'unknown': # generate preview and thumbnail for a photo try: result['previewPath'], result['thumbnailPath'], \ @@ -226,6 +227,7 @@ def _generate_photo_thumbnail_and_placeholder(self, original_file_path: str, pre placeholder for {original_file_path} {preview_file_path}: {str(exp)}') return None, None + # pylint: disable=too-many-locals def _generate_photo_preview_and_thumbnail_and_placeholder(self, original_file_path: str, mime_type: str, metadata: dict): """Generate preview and thumbnail image for a photo""" @@ -239,15 +241,28 @@ def _generate_photo_preview_and_thumbnail_and_placeholder(self, original_file_pa thumbnail_path, placeholder = self._generate_photo_thumbnail_and_placeholder( original_file_path, preview_path) return preview_path, thumbnail_path, placeholder - except Exception: - logging.warning(f'error generating preview for default photo mediaitem: {original_file_path}') - with rawpy.imread(original_file_path) as raw: - rgb = raw.postprocess(use_camera_wb=True) - img = PILImage.fromarray(rgb) - img.save(preview_path, format='JPEG') - thumbnail_path, placeholder = self._generate_photo_thumbnail_and_placeholder( - original_file_path, preview_path) - return preview_path, thumbnail_path, placeholder + except Exception as exp: + logging.warning(f'error generating preview for default \ + photo mediaitem: {original_file_path}: {str(exp)}') + try: + with open(original_file_path, 'rb') as file_reader: + if 'File:FileType'in metadata: + with WandImage(file=file_reader, format=metadata['File:FileType']) as original: + with original.convert('jpeg') as converted: + converted.save(filename=preview_path) + thumbnail_path, placeholder = self._generate_photo_thumbnail_and_placeholder( + original_file_path, preview_path) + return preview_path, thumbnail_path, placeholder + except Exception as ft_exp: + logging.warning(f'error generating preview for default \ + photo mediaitem: {original_file_path}: {str(ft_exp)}') + with rawpy.imread(original_file_path) as raw: + rgb = raw.postprocess(use_camera_wb=True) + img = PILImage.fromarray(rgb) + img.save(preview_path, format='JPEG') + thumbnail_path, placeholder = self._generate_photo_thumbnail_and_placeholder( + original_file_path, preview_path) + return preview_path, thumbnail_path, placeholder else: try: with rawpy.imread(original_file_path) as raw: @@ -257,14 +272,26 @@ def _generate_photo_preview_and_thumbnail_and_placeholder(self, original_file_pa thumbnail_path, placeholder = self._generate_photo_thumbnail_and_placeholder( original_file_path, preview_path) return preview_path, thumbnail_path, placeholder - except Exception: - logging.warning(f'error generating preview for raw photo mediaitem: {original_file_path}') - with open(original_file_path, 'rb') as file_reader: - with WandImage(file=file_reader) as original: - with original.convert('jpeg') as converted: - converted.save(filename=preview_path) - thumbnail_path, placeholder = self._generate_photo_thumbnail_and_placeholder( - original_file_path, preview_path) + except Exception as exp: + logging.warning(f'error generating preview for raw photo mediaitem: {original_file_path}: {str(exp)}') + try: + with open(original_file_path, 'rb') as file_reader: + with WandImage(file=file_reader) as original: + with original.convert('jpeg') as converted: + converted.save(filename=preview_path) + thumbnail_path, placeholder = self._generate_photo_thumbnail_and_placeholder( + original_file_path, preview_path) + except Exception as nm_exp: + logging.warning(f'error generating preview for raw \ + photo mediaitem: {original_file_path}: {str(nm_exp)}') + with open(original_file_path, 'rb') as file_reader: + if 'File:FileType'in metadata: + with WandImage(file=file_reader, format=metadata['File:FileType']) as original: + with original.convert('jpeg') as converted: + converted.save(filename=preview_path) + thumbnail_path, placeholder = self._generate_photo_thumbnail_and_placeholder( + original_file_path, preview_path) + return preview_path, thumbnail_path, placeholder return preview_path, thumbnail_path, placeholder def _generate_video_thumbnail_and_placeholder(self, preview_video_path: str): diff --git a/worker/src/providers/search/pytorch.py b/worker/src/providers/search/pytorch.py index 5df0aa8..ba453b2 100644 --- a/worker/src/providers/search/pytorch.py +++ b/worker/src/providers/search/pytorch.py @@ -26,7 +26,7 @@ def generate_embedding(self, input_type: str, data: any): logging.debug(f'generated text embedding: {res}') return res return None - if data['type'] == 'photo': + if data and data['type'] == 'photo': input_tensor = self.processor(Image.open(data['previewPath']), return_tensors='pt') res = self.vision_module.forward(**input_tensor) if res is not None: @@ -34,7 +34,7 @@ def generate_embedding(self, input_type: str, data: any): logging.debug(f'generated photo embedding: {res}') return [res] return [] - if data['type'] == 'video': + if data and data['type'] == 'video': result = [] video_clip = VideoFileClip(data['previewPath']) for frame in video_clip.iter_frames(fps=video_clip.fps):