Skip to content

Commit

Permalink
BREAKING CHANGE: Adl object refactor (#1429)
Browse files Browse the repository at this point in the history 
* Refactor objects for account and for job submission

This does the following:
* Create an explicit set of parameters for job submission. NOTE: Only
U-SQL is supported in this change, no hive jobs are supported at
present.
* Create a "Basic" jobInformation that is returned for LIST calls
* Setup inheritance for GET jobs
* Create an inheritance structure for GET and LIST ADLS and ADLA
accounts.
* This also follows the Basic<Object> -> Object inheritance pattern.

* Fix examples and job_build

* Missed some of the example changes
* Missed Job_Build, which should have its own parameters object as well.

* Moving accountId into the right place in the example.

* Updates to include name for buildjob
  • Loading branch information
@begoldsm @olydis
begoldsm authored and olydis committed Jul 18, 2017
1 parent 9556a73 commit 5f839ce
Show file tree
Hide file tree
Showing 12 changed files with 343 additions and 482 deletions.
76 changes: 2 additions & 74 deletions ...-analytics/data-plane/Microsoft.DataLakeAnalytics/2016-11-01/examples/Adla_Job_Build.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,75 +2,18 @@
"parameters": {
"api-version": "2016-11-01",
"parameters": {
"jobId": "076713da-9018-41ae-a3bd-9eab14e54d09",
"name": "test_name",
"type": "USql",
"submitter": "test_submitter",
"errorMessage": [{
"description": "test_description",
"details": "test_details",
"endOffset": 1,
"errorId": "test_error_id",
"filePath": "adl://contosoadla.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/test_file.txt",
"helpLink": "https://azure.microsoft.com/en-us/blog/introducing-azure-data-lake/",
"internalDiagnostics": "test_internal_diagnostics",
"lineNumber": 1,
"message": "test_message",
"resolution": "test_resolution",
"innerError": {
"diagnosticCode": 1,
"severity": "Warning",
"details": "test_details",
"component": "test_component",
"errorId": "test_error_id",
"helpLink": "https://azure.microsoft.com/en-us/blog/introducing-azure-data-lake/",
"internalDiagnostics": "test_internal_diagnostics",
"message": "test_message",
"resolution": "test_resolution",
"source": "SYSTEM",
"description": "test_description"
},
"severity": "Warning",
"source": "SYSTEM",
"startOffset": 1
}],
"degreeOfParallelism": 1,
"priority": 1,
"submitTime": "2017-04-18T11:16:49.0748958-07:00",
"startTime": "2017-04-18T11:16:49.0748958-07:00",
"endTime": "2017-04-18T11:16:49.0748958-07:00",
"state": "Accepted",
"result": "Succeeded",
"logFolder": "adl://contosoadla.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/logs/",
"logFilePatterns": [
"test_log_file_pattern_1",
"test_log_file_pattern_2"
],
"stateAuditRecords": [{
"newState": "test_new_state",
"timeStamp": "2017-04-18T11:16:49.0748958-07:00",
"requestedByUser": "test_requested_by_user",
"details": "test_details"
}],
"properties": {
"runtimeVersion": "test_runtime_version",
"script": "test_script",
"type": "Hive"
},
"related": {
"pipelineId": "076713da-9018-41ae-a3bd-9eab14e54d09",
"pipelineName": "test_pipeline_name",
"pipelineUri": "https://account123.contosopipelineservice.com/076713da-9018-41ae-a3bd-9eab14e54d09",
"runId": "67034c12-b250-468e-992d-39fb978bde2c",
"recurrenceId": "67034c12-b250-468e-992d-39fb978bde2d",
"recurrenceName": "test_recurrence_name"
"type": "USql"
}
}
},
"responses": {
"200": {
"body": {
"jobId": "076713da-9018-41ae-a3bd-9eab14e54d09",
"name": "test_name",
"type": "USql",
"submitter": "test_submitter",
Expand Down Expand Up @@ -102,18 +45,11 @@
"source": "SYSTEM",
"startOffset": 1
}],
"degreeOfParallelism": 1,
"priority": 1,
"submitTime": "2017-04-18T11:16:49.0748958-07:00",
"startTime": "2017-04-18T11:16:49.0748958-07:00",
"endTime": "2017-04-18T11:16:49.0748958-07:00",
"state": "Accepted",
"result": "Succeeded",
"logFolder": "adl://contosoadla.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/logs/",
"logFilePatterns": [
"test_log_file_pattern_1",
"test_log_file_pattern_2"
],
"stateAuditRecords": [{
"newState": "test_new_state",
"timeStamp": "2017-04-18T11:16:49.0748958-07:00",
Expand All @@ -123,15 +59,7 @@
"properties": {
"runtimeVersion": "test_runtime_version",
"script": "test_script",
"type": "Hive"
},
"related": {
"pipelineId": "076713da-9018-41ae-a3bd-9eab14e54d09",
"pipelineName": "test_pipeline_name",
"pipelineUri": "https://account123.contosopipelineservice.com/076713da-9018-41ae-a3bd-9eab14e54d09",
"runId": "67034c12-b250-468e-992d-39fb978bde2c",
"recurrenceId": "67034c12-b250-468e-992d-39fb978bde2d",
"recurrenceName": "test_recurrence_name"
"type": "USql"
}
}
}
Expand Down
74 changes: 2 additions & 72 deletions ...analytics/data-plane/Microsoft.DataLakeAnalytics/2016-11-01/examples/Adla_Job_Create.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,60 +3,18 @@
"api-version": "2016-11-01",
"jobIdentity": "076713da-9018-41ae-a3bd-9eab14e54d09",
"parameters": {
"jobId": "076713da-9018-41ae-a3bd-9eab14e54d09",
"name": "test_name",
"type": "USql",
"submitter": "test_submitter",
"errorMessage": [{
"description": "test_description",
"details": "test_details",
"endOffset": 1,
"errorId": "test_error_id",
"filePath": "adl://contosoadla.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/test_file.txt",
"helpLink": "https://azure.microsoft.com/en-us/blog/introducing-azure-data-lake/",
"internalDiagnostics": "test_internal_diagnostics",
"lineNumber": 1,
"message": "test_message",
"resolution": "test_resolution",
"innerError": {
"diagnosticCode": 1,
"severity": "Warning",
"details": "test_details",
"component": "test_component",
"errorId": "test_error_id",
"helpLink": "https://azure.microsoft.com/en-us/blog/introducing-azure-data-lake/",
"internalDiagnostics": "test_internal_diagnostics",
"message": "test_message",
"resolution": "test_resolution",
"source": "SYSTEM",
"description": "test_description"
},
"severity": "Warning",
"source": "SYSTEM",
"startOffset": 1
}],
"degreeOfParallelism": 1,
"priority": 1,
"submitTime": "2017-04-18T11:16:49.0748958-07:00",
"startTime": "2017-04-18T11:16:49.0748958-07:00",
"endTime": "2017-04-18T11:16:49.0748958-07:00",
"state": "Accepted",
"result": "Succeeded",
"logFolder": "adl://contosoadla.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/logs/",
"logFilePatterns": [
"test_log_file_pattern_1",
"test_log_file_pattern_2"
],
"stateAuditRecords": [{
"newState": "test_new_state",
"timeStamp": "2017-04-18T11:16:49.0748958-07:00",
"requestedByUser": "test_requested_by_user",
"details": "test_details"
}],
"properties": {
"runtimeVersion": "test_runtime_version",
"script": "test_script",
"type": "Hive"
"type": "USql"
},
"related": {
"pipelineId": "076713da-9018-41ae-a3bd-9eab14e54d09",
Expand All @@ -75,34 +33,6 @@
"name": "test_name",
"type": "USql",
"submitter": "test_submitter",
"errorMessage": [{
"description": "test_description",
"details": "test_details",
"endOffset": 1,
"errorId": "test_error_id",
"filePath": "adl://contosoadla.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/test_file.txt",
"helpLink": "https://azure.microsoft.com/en-us/blog/introducing-azure-data-lake/",
"internalDiagnostics": "test_internal_diagnostics",
"lineNumber": 1,
"message": "test_message",
"resolution": "test_resolution",
"innerError": {
"diagnosticCode": 1,
"severity": "Warning",
"details": "test_details",
"component": "test_component",
"errorId": "test_error_id",
"helpLink": "https://azure.microsoft.com/en-us/blog/introducing-azure-data-lake/",
"internalDiagnostics": "test_internal_diagnostics",
"message": "test_message",
"resolution": "test_resolution",
"source": "SYSTEM",
"description": "test_description"
},
"severity": "Warning",
"source": "SYSTEM",
"startOffset": 1
}],
"degreeOfParallelism": 1,
"priority": 1,
"submitTime": "2017-04-18T11:16:49.0748958-07:00",
Expand All @@ -124,7 +54,7 @@
"properties": {
"runtimeVersion": "test_runtime_version",
"script": "test_script",
"type": "Hive"
"type": "USql"
},
"related": {
"pipelineId": "076713da-9018-41ae-a3bd-9eab14e54d09",
Expand Down
39 changes: 0 additions & 39 deletions ...e-analytics/data-plane/Microsoft.DataLakeAnalytics/2016-11-01/examples/Adla_Job_List.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,34 +16,6 @@
"name": "test_name",
"type": "USql",
"submitter": "test_submitter",
"errorMessage": [{
"description": "test_description",
"details": "test_details",
"endOffset": 1,
"errorId": "test_error_id",
"filePath": "adl://contosoadla.azuredatalakestore.net/system/jobservice/jobs/Usql/2016/03/13/17/18/5fe51957-93bc-4de0-8ddc-c5a4753b068b/test_file.txt",
"helpLink": "https://azure.microsoft.com/en-us/blog/introducing-azure-data-lake/",
"internalDiagnostics": "test_internal_diagnostics",
"lineNumber": 1,
"message": "test_message",
"resolution": "test_resolution",
"innerError": {
"diagnosticCode": 1,
"severity": "Warning",
"details": "test_details",
"component": "test_component",
"errorId": "test_error_id",
"helpLink": "https://azure.microsoft.com/en-us/blog/introducing-azure-data-lake/",
"internalDiagnostics": "test_internal_diagnostics",
"message": "test_message",
"resolution": "test_resolution",
"source": "SYSTEM",
"description": "test_description"
},
"severity": "Warning",
"source": "SYSTEM",
"startOffset": 1
}],
"degreeOfParallelism": 1,
"priority": 1,
"submitTime": "2017-04-18T11:16:49.0748958-07:00",
Expand All @@ -56,17 +28,6 @@
"test_log_file_pattern_1",
"test_log_file_pattern_2"
],
"stateAuditRecords": [{
"newState": "test_new_state",
"timeStamp": "2017-04-18T11:16:49.0748958-07:00",
"requestedByUser": "test_requested_by_user",
"details": "test_details"
}],
"properties": {
"runtimeVersion": "test_runtime_version",
"script": "test_script",
"type": "Hive"
},
"related": {
"pipelineId": "076713da-9018-41ae-a3bd-9eab14e54d09",
"pipelineName": "test_pipeline_name",
Expand Down
Loading

0 comments on commit 5f839ce

Please sign in to comment.