Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: Query fields are sorted by creation #97

Merged
merged 2 commits into from
Jul 18, 2022
Merged

fix: Query fields are sorted by creation #97

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c485395
fix: Query fields are sorted by creation
dust1 Jul 13, 2022
6be037d
edit: remove sub mod, use impl Serialize for ResponseRows/Row
dust1 Jul 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 102 additions & 30 deletions server/src/handlers/sql.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,18 @@

//! SQL request handler

use std::collections::HashMap;

use arrow_deps::arrow::error::Result as ArrowResult;
use common_types::{datum::Datum, request_id::RequestId};
use common_types::{
datum::{Datum, DatumKind},
request_id::RequestId,
};
use interpreters::{context::Context as InterpreterContext, factory::Factory, interpreter::Output};
use log::info;
use query_engine::executor::RecordBatchVec;
use serde_derive::Serialize;
use serde::{
ser::{SerializeMap, SerializeSeq},
Serialize,
};
use snafu::ensure;
use sql::{
frontend::{Context as SqlContext, Frontend},
Expand All @@ -26,18 +30,69 @@ pub struct Request {
query: String,
}

impl From<String> for Request {
fn from(query: String) -> Self {
Self { query }
}
}

// TODO(yingwen): Improve serialize performance
#[derive(Serialize)]
#[serde(rename_all = "snake_case")]
pub enum Response {
AffectedRows(usize),
Rows(Vec<HashMap<String, Datum>>),
Rows(ResponseRows),
}

pub struct ResponseRows {
pub column_names: Vec<ResponseColumn>,
pub data: Vec<Vec<Datum>>,
}

pub struct ResponseColumn {
pub name: String,
pub data_type: DatumKind,
}

struct Row<'a>(Vec<(&'a String, &'a Datum)>);

impl<'a> Serialize for Row<'a> {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let rows = &self.0;
let mut map = serializer.serialize_map(Some(rows.len()))?;
for (key, value) in rows {
map.serialize_entry(key, value)?;
}
map.end()
}
}

impl Serialize for ResponseRows {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let total_count = self.data.len();
let mut seq = serializer.serialize_seq(Some(total_count))?;

for rows in &self.data {
let data = rows
.iter()
.enumerate()
.map(|(col_idx, datum)| {
let column_name = &self.column_names[col_idx].name;
(column_name, datum)
})
.collect::<Vec<_>>();
let row = Row(data);
seq.serialize_element(&row)?;
}

seq.end()
}
}

impl From<String> for Request {
fn from(query: String) -> Self {
Self { query }
}
}

pub async fn handle_sql<C: CatalogManager + 'static, Q: QueryExecutor + 'static>(
Expand Down Expand Up @@ -128,26 +183,43 @@ fn convert_output(output: Output) -> ArrowResult<Response> {
}

fn convert_records(records: RecordBatchVec) -> ArrowResult<Response> {
let total_rows = records.iter().map(|v| v.num_rows()).sum();
let mut resp = Vec::with_capacity(total_rows);
for record_batch in records {
let num_cols = record_batch.num_columns();
let num_rows = record_batch.num_rows();
let schema = record_batch.schema();

for row_idx in 0..num_rows {
let mut row = HashMap::with_capacity(num_cols);
for col_idx in 0..num_cols {
let column = record_batch.column(col_idx);
let column = column.datum(row_idx);

let column_name = schema.column(col_idx).name.clone();
row.insert(column_name, column);
}

resp.push(row);
if records.is_empty() {
return Ok(Response::Rows(ResponseRows {
column_names: Vec::new(),
data: Vec::new(),
}));
}

let record_batch = &records[0];
let num_cols = record_batch.num_columns();
let num_rows = record_batch.num_rows();
let schema = record_batch.schema();

let mut column_names = Vec::with_capacity(num_cols);
let mut column_data = Vec::with_capacity(num_rows);

for col_idx in 0..num_cols {
let column_schema = schema.column(col_idx).clone();
column_names.push(ResponseColumn {
name: column_schema.name,
data_type: column_schema.data_type,
});
}

for row_idx in 0..num_rows {
let mut row_data = Vec::with_capacity(num_cols);
for col_idx in 0..num_cols {
let column = record_batch.column(col_idx);
let column = column.datum(row_idx);

row_data.push(column);
}

column_data.push(row_data);
}

Ok(Response::Rows(resp))
Ok(Response::Rows(ResponseRows {
column_names,
data: column_data,
}))
}
153 changes: 88 additions & 65 deletions server/src/mysql/writer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
// Copyright 2022 CeresDB Project Authors. Licensed under Apache-2.0.

use std::collections::HashMap;

use common_types::datum::Datum;
use common_types::datum::{Datum, DatumKind};
use opensrv_mysql::{Column, ColumnFlags, ColumnType, OkResponse, QueryResultWriter};

use crate::{handlers::sql::Response, mysql::error::Result};
use crate::{
handlers::sql::{Response, ResponseColumn, ResponseRows},
mysql::error::Result,
};

pub struct MysqlQueryResultWriter<'a, W: std::io::Write> {
inner: Option<QueryResultWriter<'a, W>>,
Expand Down Expand Up @@ -35,57 +36,50 @@ impl<'a, W: std::io::Write> MysqlQueryResultWriter<'a, W> {
Ok(())
}

fn write_rows(
writer: QueryResultWriter<'a, W>,
rows: Vec<HashMap<String, Datum>>,
) -> Result<()> {
fn write_rows(writer: QueryResultWriter<'a, W>, rows: ResponseRows) -> Result<()> {
let default_response = OkResponse::default();
if rows.is_empty() {
if rows.column_names.is_empty() {
writer.completed(default_response)?;
return Ok(());
}

let columns = &rows[0]
let columns = &rows
.column_names
.iter()
.map(|(k, v)| make_column_by_field(k, v))
.map(make_column_by_field)
.collect::<Vec<_>>();
let mut row_writer = writer.start(columns)?;

for row in &rows {
for column in columns {
let key = &column.column;
if let Some(val) = row.get(key) {
let data_type = convert_field_type(val);
let re = match (data_type, val) {
(_, Datum::Varbinary(_)) => row_writer.write_col("[Varbinary]"),
(_, Datum::Null) => row_writer.write_col(None::<u8>),
(ColumnType::MYSQL_TYPE_LONG, Datum::Timestamp(t)) => {
row_writer.write_col(t.as_i64())
}
(ColumnType::MYSQL_TYPE_VARCHAR, v) => {
row_writer.write_col(v.as_str().map_or("", |s| s))
}
(ColumnType::MYSQL_TYPE_LONG, v) => {
row_writer.write_col(v.as_u64().map_or(0, |v| v))
}
(ColumnType::MYSQL_TYPE_SHORT, Datum::Boolean(b)) => {
row_writer.write_col(*b as i8)
}
(ColumnType::MYSQL_TYPE_DOUBLE, v) => {
row_writer.write_col(v.as_f64().map_or(0.0, |v| v))
}
(ColumnType::MYSQL_TYPE_FLOAT, v) => {
row_writer.write_col(v.as_f64().map_or(0.0, |v| v))
}
(_, v) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Unsupported column type, val: {:?}", v),
)),
};
re?;
} else {
row_writer.write_col(None::<u8>)?;
}
for row in &rows.data {
for val in row {
let data_type = convert_field_type(val);
let re = match (data_type, val) {
(_, Datum::Varbinary(v)) => row_writer.write_col(v.as_ref()),
(_, Datum::Null) => row_writer.write_col(None::<u8>),
(ColumnType::MYSQL_TYPE_LONG, Datum::Timestamp(t)) => {
row_writer.write_col(t.as_i64())
}
(ColumnType::MYSQL_TYPE_VARCHAR, v) => {
row_writer.write_col(v.as_str().map_or("", |s| s))
}
(ColumnType::MYSQL_TYPE_LONG, v) => {
row_writer.write_col(v.as_u64().map_or(0, |v| v))
}
(ColumnType::MYSQL_TYPE_SHORT, Datum::Boolean(b)) => {
row_writer.write_col(*b as i8)
}
(ColumnType::MYSQL_TYPE_DOUBLE, v) => {
row_writer.write_col(v.as_f64().map_or(0.0, |v| v))
}
(ColumnType::MYSQL_TYPE_FLOAT, v) => {
row_writer.write_col(v.as_f64().map_or(0.0, |v| v))
}
(_, v) => Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Unsupported column type, val: {:?}", v),
)),
};
re?;
}
row_writer.end_row()?;
}
Expand All @@ -94,16 +88,36 @@ impl<'a, W: std::io::Write> MysqlQueryResultWriter<'a, W> {
}
}

fn make_column_by_field(k: &str, v: &Datum) -> Column {
let column_type = convert_field_type(v);
fn make_column_by_field(column: &ResponseColumn) -> Column {
let column_type = conver_datum_kind_type(&column.data_type);
Column {
table: "".to_string(),
column: k.to_string(),
column: column.name.clone(),
coltype: column_type,
colflags: ColumnFlags::empty(),
}
}

fn conver_datum_kind_type(data_type: &DatumKind) -> ColumnType {
match data_type {
DatumKind::Timestamp => ColumnType::MYSQL_TYPE_LONG,
DatumKind::Double => ColumnType::MYSQL_TYPE_DOUBLE,
DatumKind::Float => ColumnType::MYSQL_TYPE_FLOAT,
DatumKind::Varbinary => ColumnType::MYSQL_TYPE_LONG_BLOB,
DatumKind::String => ColumnType::MYSQL_TYPE_VARCHAR,
DatumKind::UInt64 => ColumnType::MYSQL_TYPE_LONG,
DatumKind::UInt32 => ColumnType::MYSQL_TYPE_LONG,
DatumKind::UInt16 => ColumnType::MYSQL_TYPE_LONG,
DatumKind::UInt8 => ColumnType::MYSQL_TYPE_LONG,
DatumKind::Int64 => ColumnType::MYSQL_TYPE_LONG,
DatumKind::Int32 => ColumnType::MYSQL_TYPE_LONG,
DatumKind::Int16 => ColumnType::MYSQL_TYPE_LONG,
DatumKind::Int8 => ColumnType::MYSQL_TYPE_LONG,
DatumKind::Boolean => ColumnType::MYSQL_TYPE_SHORT,
DatumKind::Null => ColumnType::MYSQL_TYPE_NULL,
}
}

fn convert_field_type(field: &Datum) -> ColumnType {
match field {
Datum::Timestamp(_) => ColumnType::MYSQL_TYPE_LONG,
Expand All @@ -126,55 +140,64 @@ fn convert_field_type(field: &Datum) -> ColumnType {

#[cfg(test)]
mod tests {
use common_types::{datum::Datum, time::Timestamp};
use common_types::datum::DatumKind;
use opensrv_mysql::{Column, ColumnFlags, ColumnType};

use crate::mysql::writer::make_column_by_field;
use crate::{handlers::sql::ResponseColumn, mysql::writer::make_column_by_field};

struct MakeColumnTest {
k: &'static str,
v: Datum,
column: ResponseColumn,
target_type: ColumnType,
}

#[test]
fn test_make_column_by_field() {
let tests = [
MakeColumnTest {
k: "id",
v: Datum::UInt64(1),
column: ResponseColumn {
name: "id".to_string(),
data_type: DatumKind::Int32,
},
target_type: ColumnType::MYSQL_TYPE_LONG,
},
MakeColumnTest {
k: "name",
v: Datum::String("Bob".into()),
column: ResponseColumn {
name: "name".to_string(),
data_type: DatumKind::String,
},
target_type: ColumnType::MYSQL_TYPE_VARCHAR,
},
MakeColumnTest {
k: "birthday",
v: Datum::Timestamp(Timestamp::now()),
column: ResponseColumn {
name: "birthday".to_string(),
data_type: DatumKind::Timestamp,
},
target_type: ColumnType::MYSQL_TYPE_LONG,
},
MakeColumnTest {
k: "is_show",
v: Datum::Boolean(true),
column: ResponseColumn {
name: "is_show".to_string(),
data_type: DatumKind::Boolean,
},
target_type: ColumnType::MYSQL_TYPE_SHORT,
},
MakeColumnTest {
k: "money",
v: Datum::Double(12.25),
column: ResponseColumn {
name: "money".to_string(),
data_type: DatumKind::Double,
},
target_type: ColumnType::MYSQL_TYPE_DOUBLE,
},
];

for test in tests {
let target_column = Column {
table: "".to_string(),
column: test.k.to_string(),
column: test.column.name.clone(),
coltype: test.target_type,
colflags: ColumnFlags::default(),
};
assert_eq!(target_column, make_column_by_field(test.k, &test.v));
assert_eq!(target_column, make_column_by_field(&test.column));
}
}
}