Fixed task-1467

.github/workflows/unit_tests.yaml

@@ -33,11 +33,48 @@ jobs:
         run: |
           pip install -r requirements.txt
 
-      - name: Run unit tests
+      - name: Determine output folder
+        id: set_output_folder
         run: |
-          coverage run --source=src -m unittest discover -s tests/unit_tests
-          coverage xml
+          if [[ $GITHUB_EVENT_NAME == "pull_request" ]]; then
+            branch_name=$GITHUB_BASE_REF
+          else
+            branch_name=$GITHUB_REF_NAME
+          fi
+
+          if [[ $branch_name == "main" ]]; then
+            echo "output_folder=prod" >> $GITHUB_ENV
+          elif [[ $branch_name == "stage" ]]; then
+            echo "output_folder=stage" >> $GITHUB_ENV
+          elif [[ $branch_name == "dev" ]]; then
+            echo "output_folder=dev" >> $GITHUB_ENV
+          else
+            echo "Unknown branch: $branch_name"
+            exit 1
+          fi
+
+      - name: Run tests with coverage
+        run: |
+          timestamp=$(date '+%Y-%m-%d_%H-%M-%S')
+          mkdir -p test_results
+          log_file="test_results/${timestamp}_report.log"
+          echo -e "\nTest Cases Report Report\n" >> $log_file
+          # Run the tests and append output to the log file
+          python -m coverage run --source=src -m unittest discover -s tests/unit_tests >> $log_file 2>&1
+          echo -e "\nCoverage Report\n" >> $log_file
+          coverage report >> $log_file
 
       - name: Check coverage
         run: |
-          coverage report --fail-under=75
+          coverage report --fail-under=85
+
+      - name: Upload report to Azure
+        uses: LanceMcCarthy/Action-AzureBlobUpload@v2
+        with:
+          source_folder: 'test_results'
+          destination_folder: '${{ env.output_folder }}'
+          connection_string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
+          container_name: 'pathways-validation-service'
+          clean_destination_folder: false
+          delete_if_exists: false
+

.gitignore

@@ -134,4 +134,5 @@ dmypy.json
 reports/
 test_report.html
 integration_test_report.html
-/.vscode
+/.vscode
+test_results

requirements.txt

@@ -2,10 +2,10 @@ fastapi==0.88.0
 pydantic==1.10.4
 psutil~=5.9.8
 python-dotenv==1.0.1
-python-ms-core==0.0.22
+python-ms-core==0.0.23
 jsonschema==4.17.3
 httpx~=0.27.0
 coverage
 html_testRunner==1.2.1
-gtfs-canonical-validator==0.0.5
+gtfs-canonical-validator==0.0.6
 uvicorn==0.20.0

src/gtfs_pathways_validation.py

@@ -111,8 +111,8 @@ def download_single_file(self, file_upload_path=None) -> str:
         if not os.path.exists(dl_folder_path):
             os.makedirs(dl_folder_path)
 
-        file = self.storage_client.get_file_from_url(self.container_name, file_upload_path)
         try:
+            file = self.storage_client.get_file_from_url(self.container_name, file_upload_path)
             if file.file_path:
                 file_path = os.path.basename(file.file_path)
                 with open(f'{dl_folder_path}/{file_path}', 'wb') as blob:

tests/unit_tests/models/test_file_upload_msg.py

@@ -0,0 +1,92 @@
+import unittest
+from src.models.file_upload_msg import FileUploadMsg, IncomingData
+
+
+class TestFileUploadMsg(unittest.TestCase):
+
+    def test_from_dict_with_valid_data(self):
+        # Arrange
+        input_data = {
+            'messageId': '12345',
+            'messageType': 'file_upload',
+            'data': {
+                'file_upload_path': '/path/to/file',
+                'user_id': 'user_001',
+                'tdei_project_group_id': 'project_123'
+            }
+        }
+
+        # Act
+        result = FileUploadMsg.from_dict(input_data)
+
+        # Assert
+        self.assertEqual(result.messageId, '12345')
+        self.assertEqual(result.messageType, 'file_upload')
+        self.assertIsInstance(result.data, IncomingData)
+        self.assertEqual(result.data.file_upload_path, '/path/to/file')
+        self.assertEqual(result.data.user_id, 'user_001')
+        self.assertEqual(result.data.tdei_project_group_id, 'project_123')
+
+    def test_from_dict_with_partial_data(self):
+        # Arrange
+        input_data = {
+            'messageId': '12345',
+            'messageType': 'file_upload',
+            'data': {
+                'file_upload_path': '/path/to/file',
+                'user_id': 'user_001'
+            }
+        }
+
+        # Act
+        result = FileUploadMsg.from_dict(input_data)
+
+        # Assert
+        self.assertEqual(result.messageId, '12345')
+        self.assertEqual(result.messageType, 'file_upload')
+        self.assertIsInstance(result.data, IncomingData)
+        self.assertEqual(result.data.file_upload_path, '/path/to/file')
+        self.assertEqual(result.data.user_id, 'user_001')
+        self.assertEqual(result.data.tdei_project_group_id, '')
+
+    def test_from_dict_with_no_data_key(self):
+        # Arrange
+        input_data = {
+            'messageId': '12345',
+            'messageType': 'file_upload'
+        }
+
+        # Act
+        result = FileUploadMsg.from_dict(input_data)
+
+        # Assert
+        self.assertEqual(result.messageId, '12345')
+        self.assertEqual(result.messageType, 'file_upload')
+        self.assertIsNone(result.data)
+
+    def test_from_dict_with_empty_dict(self):
+        # Arrange
+        input_data = {}
+
+        # Act
+        result = FileUploadMsg.from_dict(input_data)
+
+        # Assert
+        self.assertIsNone(result.messageId)
+        self.assertIsNone(result.messageType)
+        self.assertIsNone(result.data)
+
+    def test_incoming_data_default_tdei_project_group_id(self):
+        # Arrange
+        incoming_data = IncomingData(
+            file_upload_path='/path/to/file',
+            user_id='user_001'
+        )
+
+        # Assert
+        self.assertEqual(incoming_data.file_upload_path, '/path/to/file')
+        self.assertEqual(incoming_data.user_id, 'user_001')
+        self.assertEqual(incoming_data.tdei_project_group_id, '')
+
+if __name__ == '__main__':
+    unittest.main()

tests/unit_tests/serializer/test_gtfs_pathways_serializer.py

@@ -0,0 +1,108 @@
+import unittest
+from src.serializer.gtfs_pathways_serializer import (
+    GTFSPathwaysUpload,
+    GTFSPathwaysUploadData,
+    Request,
+    Meta,
+    Response,
+    remove_underscore
+)
+
+class TestGTFSPathwaysUpload(unittest.TestCase):
+    def test_gtfs_pathways_upload_initialization_with_data(self):
+        data = {
+            'messageId': '12345',
+            'messageType': 'file_upload',
+            'message': 'Test message',
+            'publishedDate': '2024-11-20',
+            'data': {
+                'stage': 'stage1',
+                'tdei_record_id': 'record123',
+                'tdei_project_group_id': 'group123',
+                'user_id': 'user123',
+                'request': {
+                    'tdei_project_group_id': 'group123',
+                    'tdei_station_id': 'station123',
+                    'collected_by': 'user123',
+                    'collection_date': '2024-11-20',
+                    'collection_method': 'method1',
+                    'valid_from': '2024-11-20',
+                    'valid_to': '2024-11-21',
+                    'data_source': 'source1',
+                    'polygon': {},
+                    'pathways_schema_version': 'v1.0',
+                },
+                'meta': {'file_upload_path': '/path/to/file'},
+                'response': {'success': True, 'message': 'Validation successful'},
+            },
+        }
+        upload = GTFSPathwaysUpload(data=data)
+
+        self.assertEqual(upload.message_id, '12345')
+        self.assertEqual(upload.message_type, 'file_upload')
+        self.assertEqual(upload.message, 'Test message')
+        self.assertEqual(upload.published_date, '2024-11-20')
+        self.assertIsInstance(upload.data, GTFSPathwaysUploadData)
+        self.assertEqual(upload.data.stage, 'stage1')
+        self.assertEqual(upload.data.tdei_record_id, 'record123')
+        self.assertEqual(upload.data.tdei_project_group_id, 'group123')
+        self.assertEqual(upload.data.user_id, 'user123')
+
+    def test_gtfs_pathways_upload_initialization_without_data(self):
+        data = {}
+        upload = GTFSPathwaysUpload(data=data)
+
+        self.assertEqual(upload.message_id, '')
+        self.assertIsNone(upload.message_type)
+        self.assertIsNone(upload.message)
+        self.assertIsNone(upload.published_date)
+        self.assertEqual(upload.data, {})
+
+
+    def test_remove_underscore(self):
+        self.assertEqual(remove_underscore('_fieldName'), 'fieldName')
+        self.assertEqual(remove_underscore('fieldName'), 'fieldName')
+
+    def test_request_initialization(self):
+        request_data = {
+            'tdei_project_group_id': 'group123',
+            'tdei_station_id': 'station123',
+            'collected_by': 'user123',
+            'collection_date': '2024-11-20',
+            'collection_method': 'method1',
+            'valid_from': '2024-11-20',
+            'valid_to': '2024-11-21',
+            'data_source': 'source1',
+            'polygon': {},
+            'pathways_schema_version': 'v1.0',
+        }
+        request = Request(data=request_data)
+
+        self.assertEqual(request.tdei_project_group_id, 'group123')
+        self.assertEqual(request.tdei_station_id, 'station123')
+        self.assertEqual(request.collected_by, 'user123')
+        self.assertEqual(request.collection_date, '2024-11-20')
+        self.assertEqual(request.collection_method, 'method1')
+        self.assertEqual(request.valid_from, '2024-11-20')
+        self.assertEqual(request.valid_to, '2024-11-21')
+        self.assertEqual(request.data_source, 'source1')
+        self.assertEqual(request.pathways_schema_version, 'v1.0')
+
+    def test_meta_initialization(self):
+        meta_data = {'file_upload_path': '/path/to/file'}
+        meta = Meta(data=meta_data)
+
+        self.assertEqual(meta.file_upload_path, '/path/to/file')
+        self.assertFalse(meta.isValid)
+        self.assertEqual(meta.validationMessage, '')
+
+    def test_response_initialization(self):
+        response_data = {'success': True, 'message': 'Validation successful'}
+        response = Response(data=response_data)
+
+        self.assertTrue(response.success)
+        self.assertEqual(response.message, 'Validation successful')
+
+
+if __name__ == '__main__':
+    unittest.main()