|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +//! Line-delimited JSON data source |
| 19 | +//! |
| 20 | +//! This data source allows Line-delimited JSON string or files to be used as input for queries. |
| 21 | +//! |
| 22 | +
|
| 23 | +use std::{ |
| 24 | + any::Any, |
| 25 | + io::{BufReader, Read, Seek}, |
| 26 | + sync::{Arc, Mutex}, |
| 27 | +}; |
| 28 | + |
| 29 | +use crate::{ |
| 30 | + datasource::{Source, TableProvider}, |
| 31 | + error::{DataFusionError, Result}, |
| 32 | + physical_plan::{ |
| 33 | + common, |
| 34 | + json::{NdJsonExec, NdJsonReadOptions}, |
| 35 | + ExecutionPlan, |
| 36 | + }, |
| 37 | +}; |
| 38 | +use arrow::{datatypes::SchemaRef, json::reader::infer_json_schema_from_seekable}; |
| 39 | + |
| 40 | +use super::datasource::Statistics; |
| 41 | + |
| 42 | +trait SeekRead: Read + Seek {} |
| 43 | + |
| 44 | +impl<T: Seek + Read> SeekRead for T {} |
| 45 | + |
| 46 | +/// Represents a line-delimited JSON file with a provided schema |
| 47 | +pub struct NdJsonFile { |
| 48 | + source: Source<Box<dyn SeekRead + Send + Sync + 'static>>, |
| 49 | + schema: SchemaRef, |
| 50 | + file_extension: String, |
| 51 | + statistics: Statistics, |
| 52 | +} |
| 53 | + |
| 54 | +impl NdJsonFile { |
| 55 | + /// Attempt to initialize a `NdJsonFile` from a path. The schema can be inferred automatically. |
| 56 | + pub fn try_new(path: &str, options: NdJsonReadOptions) -> Result<Self> { |
| 57 | + let schema = if let Some(schema) = options.schema { |
| 58 | + schema |
| 59 | + } else { |
| 60 | + let filenames = common::build_file_list(path, options.file_extension)?; |
| 61 | + if filenames.is_empty() { |
| 62 | + return Err(DataFusionError::Plan(format!( |
| 63 | + "No files found at {path} with file extension {file_extension}", |
| 64 | + path = path, |
| 65 | + file_extension = options.file_extension |
| 66 | + ))); |
| 67 | + } |
| 68 | + |
| 69 | + NdJsonExec::try_infer_schema( |
| 70 | + filenames, |
| 71 | + Some(options.schema_infer_max_records), |
| 72 | + )? |
| 73 | + .into() |
| 74 | + }; |
| 75 | + |
| 76 | + Ok(Self { |
| 77 | + source: Source::Path(path.to_string()), |
| 78 | + schema, |
| 79 | + file_extension: options.file_extension.to_string(), |
| 80 | + statistics: Statistics::default(), |
| 81 | + }) |
| 82 | + } |
| 83 | + |
| 84 | + /// Attempt to initialize a `NdJsonFile` from a reader impls `Seek`. The schema can be inferred automatically. |
| 85 | + pub fn try_new_from_reader<R: Read + Seek + Send + Sync + 'static>( |
| 86 | + mut reader: R, |
| 87 | + options: NdJsonReadOptions, |
| 88 | + ) -> Result<Self> { |
| 89 | + let schema = if let Some(schema) = options.schema { |
| 90 | + schema |
| 91 | + } else { |
| 92 | + let mut bufr = BufReader::new(reader); |
| 93 | + let schema = infer_json_schema_from_seekable( |
| 94 | + &mut bufr, |
| 95 | + Some(options.schema_infer_max_records), |
| 96 | + )? |
| 97 | + .into(); |
| 98 | + reader = bufr.into_inner(); |
| 99 | + schema |
| 100 | + }; |
| 101 | + Ok(Self { |
| 102 | + source: Source::Reader(Mutex::new(Some(Box::new(reader)))), |
| 103 | + schema, |
| 104 | + statistics: Statistics::default(), |
| 105 | + file_extension: String::new(), |
| 106 | + }) |
| 107 | + } |
| 108 | +} |
| 109 | +impl TableProvider for NdJsonFile { |
| 110 | + fn as_any(&self) -> &dyn Any { |
| 111 | + self |
| 112 | + } |
| 113 | + |
| 114 | + fn schema(&self) -> SchemaRef { |
| 115 | + self.schema.clone() |
| 116 | + } |
| 117 | + |
| 118 | + fn scan( |
| 119 | + &self, |
| 120 | + projection: &Option<Vec<usize>>, |
| 121 | + batch_size: usize, |
| 122 | + _filters: &[crate::logical_plan::Expr], |
| 123 | + limit: Option<usize>, |
| 124 | + ) -> Result<Arc<dyn ExecutionPlan>> { |
| 125 | + let opts = NdJsonReadOptions { |
| 126 | + schema: Some(self.schema.clone()), |
| 127 | + schema_infer_max_records: 0, // schema will always be provided, so it's unnecessary to infer schema |
| 128 | + file_extension: self.file_extension.as_str(), |
| 129 | + }; |
| 130 | + let batch_size = limit |
| 131 | + .map(|l| std::cmp::min(l, batch_size)) |
| 132 | + .unwrap_or(batch_size); |
| 133 | + |
| 134 | + let exec = match &self.source { |
| 135 | + Source::Reader(maybe_reader) => { |
| 136 | + if let Some(rdr) = maybe_reader.lock().unwrap().take() { |
| 137 | + NdJsonExec::try_new_from_reader( |
| 138 | + rdr, |
| 139 | + opts, |
| 140 | + projection.clone(), |
| 141 | + batch_size, |
| 142 | + limit, |
| 143 | + )? |
| 144 | + } else { |
| 145 | + return Err(DataFusionError::Execution( |
| 146 | + "You can only read once if the data comes from a reader" |
| 147 | + .to_string(), |
| 148 | + )); |
| 149 | + } |
| 150 | + } |
| 151 | + Source::Path(p) => { |
| 152 | + NdJsonExec::try_new(&p, opts, projection.clone(), batch_size, limit)? |
| 153 | + } |
| 154 | + }; |
| 155 | + Ok(Arc::new(exec)) |
| 156 | + } |
| 157 | + |
| 158 | + fn statistics(&self) -> Statistics { |
| 159 | + self.statistics.clone() |
| 160 | + } |
| 161 | +} |
| 162 | + |
| 163 | +#[cfg(test)] |
| 164 | +mod tests { |
| 165 | + use super::*; |
| 166 | + use crate::prelude::*; |
| 167 | + const TEST_DATA_BASE: &str = "tests/jsons"; |
| 168 | + |
| 169 | + #[tokio::test] |
| 170 | + async fn csv_file_from_reader() -> Result<()> { |
| 171 | + let mut ctx = ExecutionContext::new(); |
| 172 | + let path = format!("{}/2.json", TEST_DATA_BASE); |
| 173 | + ctx.register_table( |
| 174 | + "ndjson", |
| 175 | + Arc::new(NdJsonFile::try_new(&path, Default::default())?), |
| 176 | + )?; |
| 177 | + let df = ctx.sql("select sum(a) from ndjson")?; |
| 178 | + let batches = df.collect().await?; |
| 179 | + assert_eq!( |
| 180 | + batches[0] |
| 181 | + .column(0) |
| 182 | + .as_any() |
| 183 | + .downcast_ref ::<arrow ::array ::Int64Array> () |
| 184 | + .unwrap() |
| 185 | + .value(0), |
| 186 | + 100000000000011 |
| 187 | + ); |
| 188 | + Ok(()) |
| 189 | + } |
| 190 | +} |
0 commit comments