Skip to content

Conversation

@stephenplusplus
Copy link
Contributor

This adds tests as suggested here: #1854 (comment)

It also caught a bug where we were promisifying the bigquery.date() (and related) methods.

@stephenplusplus stephenplusplus added the api: bigquery Issues related to the BigQuery API. label Dec 7, 2016
@googlebot googlebot added the cla: yes This human has signed the Contributor License Agreement. label Dec 7, 2016
@stephenplusplus
Copy link
Contributor Author

@c0b -- please take a look and let me know if this is what you had in mind. Also, if you can show a query that echos a STRUCT, that would be great as well. Thanks!

@c0b
Copy link
Contributor

c0b commented Dec 7, 2016

ok, from https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types I can figure out this following example:

BTW: you can see the bq tool is using google-api-python-client/1.3.1 library but not doing any data type restoration, all primitive types (integer, float, boolean, ...) turns to a string in the prettyjson output, the bq tool comes as part of google-cloud-sdk maybe someone can file an issue there

$ bq --apilog - query \
   --use_legacy_sql=false --format=prettyjson \
   --parameter obj:'STRUCT<b BOOL, arr ARRAY<INTEGER>,
      d DATETIME, f FLOAT64, nested STRUCT<a INT64>,
      nullable INT64>':'
      { "b": true, "arr": [2,3,4], "d": "2016-11-6", "f": 3.14, "nested": {"a": 3} }' \
   'SELECT @obj obj'

[...]
INFO:root:{
 "kind": "bigquery#job",
 "configuration": {
  "query": {
   "query": "SELECT @obj obj",
   "createDisposition": "CREATE_IF_NEEDED",
   "writeDisposition": "WRITE_TRUNCATE",
   "useLegacySql": false,
   "queryParameters": [
    {
     "name": "obj",
     "parameterType": {
      "type": "STRUCT",
      "structTypes": [
       {
        "name": "b",
        "type": {
         "type": "BOOL"
        }
       },
       {
        "name": "arr",
        "type": {
         "type": "ARRAY",
         "arrayType": {
          "type": "INT64"
         }
        }
       },
       {
        "name": "d",
        "type": {
         "type": "DATETIME"
        }
       },
       {
        "name": "f",
        "type": {
         "type": "FLOAT64"
        }
       },
       {
        "name": "nested",
        "type": {
         "type": "STRUCT",
         "structTypes": [
          {
           "name": "a",
           "type": {
            "type": "INT64"
           }
          }
         ]
        }
       },
       {
        "name": "nullable",
        "type": {
         "type": "INT64"
        }
       }
      ]
     },
     "parameterValue": {
      "structValues": {
       "arr": {
        "arrayValues": [
         {
          "value": "2"
         },
         {
          "value": "3"
         },
         {
          "value": "4"
         }
        ]
       },
       "b": {
        "value": "true"
       },
       "d": {
        "value": "2016-11-6"
       },
       "f": {
        "value": "3.14"
       },
       "nested": {
        "structValues": {
         "a": {
          "value": "3"
         }
        }
       }
      }
     }
    }
   ]
  }
 },
 "status": {
  "state": "RUNNING"
 },
}

========== RESPONSE =============
INFO:root:{
 "kind": "bigquery#job",
 "configuration": {
  "query": {
   "query": "SELECT @obj obj",
   "createDisposition": "CREATE_IF_NEEDED",
   "writeDisposition": "WRITE_TRUNCATE",
   "useLegacySql": false,
   "queryParameters": [
    {
     "name": "obj",
     "parameterType": {
      "type": "STRUCT",
      "structTypes": [
       {
        "name": "b",
        "type": {
         "type": "BOOL"
        }
       },
       {
        "name": "arr",
        "type": {
         "type": "ARRAY",
         "arrayType": {
          "type": "INT64"
         }
        }
       },
       {
        "name": "d",
        "type": {
         "type": "DATETIME"
        }
       },
       {
        "name": "f",
        "type": {
         "type": "FLOAT64"
        }
       },
       {
        "name": "nested",
        "type": {
         "type": "STRUCT",
         "structTypes": [
          {
           "name": "a",
           "type": {
            "type": "INT64"
           }
          }
         ]
        }
       },
       {
        "name": "nullable",
        "type": {
         "type": "INT64"
        }
       }
      ]
     },
     "parameterValue": {
      "structValues": {
       "arr": {
        "arrayValues": [
         {
          "value": "2"
         },
         {
          "value": "3"
         },
         {
          "value": "4"
         }
        ]
       },
       "b": {
        "value": "true"
       },
       "d": {
        "value": "2016-11-6"
       },
       "f": {
        "value": "3.14"
       },
       "nested": {
        "structValues": {
         "a": {
          "value": "3"
         }
        }
       }
      }
     }
    }
   ]
  }
 },
 "status": {
  "state": "DONE"
 },
 "statistics": {
  "creationTime": "1481134602568",
  "startTime": "1481134602697",
  "endTime": "1481134604540",
  "totalBytesProcessed": "0",
  "query": {
   "queryPlan": [
    {
     "name": "Stage 1",
     "id": "1",
     "waitRatioAvg": 0.8185889148520366,
     "waitRatioMax": 0.8185889148520366,
     "readRatioAvg": 0.0,
     "readRatioMax": 0.0,
     "computeRatioAvg": 0.07351883908714532,
     "computeRatioMax": 0.07351883908714532,
     "writeRatioAvg": 1.0,
     "writeRatioMax": 1.0,
     "recordsRead": "0",
     "recordsWritten": "1",
     "steps": [
      {
       "kind": "COMPUTE",
       "substeps": [
        "STRUCT\u003c...\u003e"
       ]
      },
      {
       "kind": "COMPUTE",
       "substeps": [
        "STRUCT\u003c...\u003e"
       ]
      },
      {
       "kind": "WRITE",
       "substeps": [
        "$1",
        "TO __output"
       ]
      }
     ]
    }
   ],
   "totalBytesProcessed": "0",
   "totalBytesBilled": "0",
   "billingTier": 1,
   "cacheHit": false
  }
 },
}

INFO:root:--response-end--
Waiting on xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ... (1s) Current status: DONE   
INFO:root:--request-start--
INFO:root:-headers-start-
INFO:root:accept-encoding: gzip, deflate
INFO:root:accept: application/json
INFO:root:user-agent: google-api-python-client/1.3.1 (gzip)
INFO:root:-headers-end-
INFO:root:-path-parameters-start-
INFO:root:-path-parameters-end-
INFO:root:body: None
INFO:root:query: ?timeoutMs=0&startIndex=0&alt=json&maxResults=100
INFO:root:--request-end--
INFO:root:--response-start--
INFO:root:status: 200
INFO:root:content-length: 1628
INFO:root:x-xss-protection: 1; mode=block
INFO:root:x-content-type-options: nosniff
INFO:root:transfer-encoding: chunked
INFO:root:expires: Wed, 07 Dec 2016 18:16:45 GMT
INFO:root:vary: Origin, X-Origin
INFO:root:server: GSE
INFO:root:{
 "kind": "bigquery#getQueryResultsResponse",
 "schema": {
  "fields": [
   {
    "name": "obj",
    "type": "RECORD",
    "mode": "NULLABLE",
    "fields": [
     {
      "name": "b",
      "type": "BOOLEAN",
      "mode": "NULLABLE"
     },
     {
      "name": "arr",
      "type": "INTEGER",
      "mode": "REPEATED"
     },
     {
      "name": "d",
      "type": "DATETIME",
      "mode": "NULLABLE"
     },
     {
      "name": "f",
      "type": "FLOAT",
      "mode": "NULLABLE"
     },
     {
      "name": "nested",
      "type": "RECORD",
      "mode": "NULLABLE",
      "fields": [
       {
        "name": "a",
        "type": "INTEGER",
        "mode": "NULLABLE"
       }
      ]
     },
     {
      "name": "nullable",
      "type": "INTEGER",
      "mode": "NULLABLE"
     }
    ]
   }
  ]
 },
 "totalRows": "1",
 "rows": [
  {
   "f": [
    {
     "v": {
      "f": [
       {
        "v": "true"
       },
       {
        "v": [
         {
          "v": "2"
         },
         {
          "v": "3"
         },
         {
          "v": "4"
         }
        ]
       },
       {
        "v": "2016-11-06T00:00:00"
       },
       {
        "v": "3.14"
       },
       {
        "v": {
         "f": [
          {
           "v": "3"
          }
         ]
        }
       },
       {
        "v": null
       }
      ]
     }
    }
   ]
  }
 ],
 "totalBytesProcessed": "0",
 "jobComplete": true,
 "cacheHit": false
}

INFO:root:--response-end--
[
  {
    "obj": {
      "arr": [
        "2", 
        "3", 
        "4"
      ], 
      "b": "true", 
      "d": "2016-11-06T00:00:00", 
      "f": "3.14", 
      "nested": {
        "a": "3"
      }, 
      "nullable": null
    }
  }
]

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

api: bigquery Issues related to the BigQuery API. cla: yes This human has signed the Contributor License Agreement.

Projects

None yet

Development

Successfully merging this pull request may close these issues.

4 participants