Skip to content

feat(cubesql): Fill pg_description table with cube and members descriptions #8618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 6, 2024
Merged

feat(cubesql): Fill pg_description table with cube and members descriptions #8618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d86d7c3
feat(cubesql): Add description to CubeMetaTable and CubeMetaColumn
mcheshkov Aug 17, 2024
99df4f7
feat(cubesql): Fill pg_description table with cube descriptions
mcheshkov Aug 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion rust/cubesql/cubesql/src/compile/engine/context_postgresql.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,7 +327,11 @@ impl DatabaseProtocol {
context.session_state.all_variables(),
)))
}
"pg_description" => return Some(Arc::new(PgCatalogDescriptionProvider::new())),
"pg_description" => {
return Some(Arc::new(PgCatalogDescriptionProvider::new(
&context.meta.tables,
)))
}
"pg_constraint" => return Some(Arc::new(PgCatalogConstraintProvider::new())),
"pg_depend" => return Some(Arc::new(PgCatalogDependProvider::new())),
"pg_am" => return Some(Arc::new(PgCatalogAmProvider::new())),
Expand Down
44 changes: 41 additions & 3 deletions rust/cubesql/cubesql/src/compile/engine/information_schema/postgres/pg_description.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::{any::Any, sync::Arc};
use std::{any::Any, convert::TryFrom, sync::Arc};

use async_trait::async_trait;

Expand All @@ -14,10 +14,19 @@ use datafusion::{
physical_plan::{memory::MemoryExec, ExecutionPlan},
};

use crate::{
compile::engine::information_schema::postgres::PG_CLASS_CLASS_OID, transport::CubeMetaTable,
};

/// See https://www.postgresql.org/docs/16/catalog-pg-description.html
struct PgCatalogDescriptionBuilder {
/// The OID of the object this description pertains to
objoid: UInt32Builder,
/// The OID of the system catalog this object appears in
classoid: UInt32Builder,
/// For a comment on a table column, this is the column number (the objoid and classoid refer to the table itself). For all other object types, this column is zero.
objsubid: Int32Builder,
/// Arbitrary text that serves as the description of this object
description: StringBuilder,
}

Expand All @@ -33,6 +42,23 @@ impl PgCatalogDescriptionBuilder {
}
}

fn add_table(&mut self, table_oid: u32, description: impl AsRef<str>) {
self.objoid.append_value(table_oid).unwrap();
self.classoid.append_value(PG_CLASS_CLASS_OID).unwrap();
self.objsubid.append_value(0).unwrap();
self.description.append_value(description).unwrap();
}

fn add_column(&mut self, table_oid: u32, column_idx: usize, description: impl AsRef<str>) {
self.objoid.append_value(table_oid).unwrap();
self.classoid.append_value(PG_CLASS_CLASS_OID).unwrap();
// Column subids starts with 1
self.objsubid
.append_value(i32::try_from(column_idx).unwrap() + 1)
.unwrap();
self.description.append_value(description).unwrap();
}

fn finish(mut self) -> Vec<Arc<dyn Array>> {
let columns: Vec<Arc<dyn Array>> = vec![
Arc::new(self.objoid.finish()),
Expand All @@ -50,8 +76,20 @@ pub struct PgCatalogDescriptionProvider {
}

impl PgCatalogDescriptionProvider {
pub fn new() -> Self {
let builder = PgCatalogDescriptionBuilder::new();
pub fn new(tables: &[CubeMetaTable]) -> Self {
let mut builder = PgCatalogDescriptionBuilder::new();

for table in tables {
if let Some(description) = &table.description {
builder.add_table(table.oid, description);
}

for (idx, column) in table.columns.iter().enumerate() {
if let Some(description) = &column.description {
builder.add_column(table.oid, idx, description);
}
}
}

Self {
data: Arc::new(builder.finish()),
Expand Down
40 changes: 20 additions & 20 deletions ...besql/cubesql/src/compile/snapshots/cubesql__compile__tests__metabase_pg_class_query.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,23 @@
source: cubesql/src/compile/mod.rs
expression: "execute_query(\"\n SELECT *\n FROM (\n SELECT n.nspname,\n c.relname,\n a.attname,\n a.atttypid,\n a.attnotnull or (t.typtype = 'd' AND t.typnotnull) AS attnotnull,\n a.atttypmod,\n a.attlen,\n t.typtypmod,\n row_number() OVER (partition BY a.attrelid ORDER BY a.attnum) AS attnum,\n NULLIF(a.attidentity, '') AS attidentity,\n pg_catalog.pg_get_expr(def.adbin, def.adrelid) AS adsrc,\n dsc.description,\n t.typbasetype,\n t.typtype\n FROM pg_catalog.pg_namespace n\n JOIN pg_catalog.pg_class c ON (c.relnamespace = n.oid)\n JOIN pg_catalog.pg_attribute a ON (a.attrelid=c.oid)\n JOIN pg_catalog.pg_type t ON (a.atttypid = t.oid)\n LEFT JOIN pg_catalog.pg_attrdef def ON (a.attrelid=def.adrelid AND a.attnum = def.adnum)\n LEFT JOIN pg_catalog.pg_description dsc ON (c.oid=dsc.objoid AND a.attnum = dsc.objsubid)\n LEFT JOIN pg_catalog.pg_class dc ON (dc.oid=dsc.classoid AND dc.relname='pg_class')\n LEFT JOIN pg_catalog.pg_namespace dn ON (dc.relnamespace=dn.oid AND dn.nspname='pg_catalog')\n WHERE c.relkind IN ('r', 'p', 'v', 'f', 'm') AND a.attnum > 0 AND NOT a.attisdropped AND n.nspname LIKE 'public' AND c.relname LIKE 'KibanaSampleDataEcommerce') c\n WHERE true\n ORDER BY nspname, c.relname, attnum;\n \".to_string(),\nDatabaseProtocol::PostgreSQL).await?"
---
+---------+---------------------------+--------------------+----------+------------+-----------+--------+-----------+--------+-------------+-------+-------------+-------------+---------+
| nspname | relname | attname | atttypid | attnotnull | atttypmod | attlen | typtypmod | attnum | attidentity | adsrc | description | typbasetype | typtype |
+---------+---------------------------+--------------------+----------+------------+-----------+--------+-----------+--------+-------------+-------+-------------+-------------+---------+
| public | KibanaSampleDataEcommerce | count | 20 | true | -1 | 8 | -1 | 1 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | maxPrice | 1700 | true | -1 | -1 | -1 | 2 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | sumPrice | 1700 | true | -1 | -1 | -1 | 3 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | minPrice | 1700 | true | -1 | -1 | -1 | 4 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | avgPrice | 1700 | true | -1 | -1 | -1 | 5 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | countDistinct | 20 | true | -1 | 8 | -1 | 6 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | order_date | 1114 | false | -1 | 8 | -1 | 7 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | last_mod | 1114 | false | -1 | 8 | -1 | 8 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | customer_gender | 25 | false | -1 | -1 | -1 | 9 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | notes | 25 | false | -1 | -1 | -1 | 10 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | taxful_total_price | 1700 | false | -1 | -1 | -1 | 11 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | has_subscription | 16 | false | -1 | 1 | -1 | 12 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | is_male | 16 | true | -1 | 1 | -1 | 13 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | is_female | 16 | true | -1 | 1 | -1 | 14 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | __user | 25 | false | -1 | -1 | -1 | 15 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | __cubeJoinField | 25 | false | -1 | -1 | -1 | 16 | NULL | NULL | NULL | 0 | b |
+---------+---------------------------+--------------------+----------+------------+-----------+--------+-----------+--------+-------------+-------+-------------+-------------+---------+
+---------+---------------------------+--------------------+----------+------------+-----------+--------+-----------+--------+-------------+-------+-----------------------------------------------+-------------+---------+
| nspname | relname | attname | atttypid | attnotnull | atttypmod | attlen | typtypmod | attnum | attidentity | adsrc | description | typbasetype | typtype |
+---------+---------------------------+--------------------+----------+------------+-----------+--------+-----------+--------+-------------+-------+-----------------------------------------------+-------------+---------+
| public | KibanaSampleDataEcommerce | count | 20 | true | -1 | 8 | -1 | 1 | NULL | NULL | Events count | 0 | b |
| public | KibanaSampleDataEcommerce | maxPrice | 1700 | true | -1 | -1 | -1 | 2 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | sumPrice | 1700 | true | -1 | -1 | -1 | 3 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | minPrice | 1700 | true | -1 | -1 | -1 | 4 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | avgPrice | 1700 | true | -1 | -1 | -1 | 5 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | countDistinct | 20 | true | -1 | 8 | -1 | 6 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | order_date | 1114 | false | -1 | 8 | -1 | 7 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | last_mod | 1114 | false | -1 | 8 | -1 | 8 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | customer_gender | 25 | false | -1 | -1 | -1 | 9 | NULL | NULL | Customer gender | 0 | b |
| public | KibanaSampleDataEcommerce | notes | 25 | false | -1 | -1 | -1 | 10 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | taxful_total_price | 1700 | false | -1 | -1 | -1 | 11 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | has_subscription | 16 | false | -1 | 1 | -1 | 12 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | is_male | 16 | true | -1 | 1 | -1 | 13 | NULL | NULL | Male users segment | 0 | b |
| public | KibanaSampleDataEcommerce | is_female | 16 | true | -1 | 1 | -1 | 14 | NULL | NULL | NULL | 0 | b |
| public | KibanaSampleDataEcommerce | __user | 25 | false | -1 | -1 | -1 | 15 | NULL | NULL | Virtual column for security context switching | 0 | b |
| public | KibanaSampleDataEcommerce | __cubeJoinField | 25 | false | -1 | -1 | -1 | 16 | NULL | NULL | Virtual column for joining cubes | 0 | b |
+---------+---------------------------+--------------------+----------+------------+-----------+--------+-----------+--------+-------------+-------+-----------------------------------------------+-------------+---------+
Loading