diff --git a/src/main/mondrian/rolap/agg/AggregationManager.java b/src/main/mondrian/rolap/agg/AggregationManager.java index 437722724e..be3fb90c6a 100644 --- a/src/main/mondrian/rolap/agg/AggregationManager.java +++ b/src/main/mondrian/rolap/agg/AggregationManager.java @@ -5,12 +5,11 @@ // You must accept the terms of that agreement to use this software. // // Copyright (C) 2001-2005 Julian Hyde -// Copyright (C) 2005-2013 Pentaho and others +// Copyright (C) 2005-2014 Pentaho and others // All Rights Reserved. // // jhyde, 30 August, 2001 */ - package mondrian.rolap.agg; import mondrian.olap.CacheControl; @@ -378,9 +377,19 @@ public static AggStar findAgg( // the agg stars levels, or if the agg star is not // fully collapsed. rollup[0] = !aggStar.isFullyCollapsed() + || aggStar.hasIgnoredColumns() || (levelBitKey.isEmpty() || !aggStar.getLevelBitKey().equals(levelBitKey)); return aggStar; + } else if (aggStar.hasIgnoredColumns()) { + // we cannot safely pull a distinct count from an agg + // table if ignored columns are present since granularity + // may not be at the level of the dc measure + LOGGER.info( + aggStar.getFactTable().getName() + + " cannot be used for distinct-count measures since it has" + + " unused or ignored columns."); + continue; } // If there are distinct measures, we can only rollup in limited diff --git a/src/main/mondrian/rolap/aggmatcher/AggStar.java b/src/main/mondrian/rolap/aggmatcher/AggStar.java index 191eebf525..1bd759f326 100644 --- a/src/main/mondrian/rolap/aggmatcher/AggStar.java +++ b/src/main/mondrian/rolap/aggmatcher/AggStar.java @@ -5,10 +5,9 @@ // You must accept the terms of that agreement to use this software. // // Copyright (C) 2005-2005 Julian Hyde -// Copyright (C) 2005-2013 Pentaho and others +// Copyright (C) 2005-2014 Pentaho and others // All Rights Reserved. */ - package mondrian.rolap.aggmatcher; import mondrian.olap.*; @@ -55,6 +54,7 @@ */ public class AggStar { private static final Logger LOGGER = Logger.getLogger(AggStar.class); + private boolean hasIgnoredColumns; static Logger getLogger() { return LOGGER; @@ -110,6 +110,8 @@ public static AggStar makeAggStar( JdbcSchema.Table.Column.Usage usage = it.next(); aggStarFactTable.loadLevel(usage); } + aggStar.hasIgnoredColumns = + dbTable.getColumnUsages(UsageType.IGNORE).hasNext(); // 5. for each distinct-count measure, populate a list of the levels // which it is OK to roll up @@ -450,6 +452,10 @@ private void addColumn(final AggStar.Table.Column column) { private static final Logger JOIN_CONDITION_LOGGER = Logger.getLogger(AggStar.Table.JoinCondition.class); + public boolean hasIgnoredColumns() { + return hasIgnoredColumns; + } + /** * Base Table class for the FactTable and DimTable classes. * This class parallels the RolapStar.Table class. diff --git a/src/main/mondrian/rolap/aggmatcher/Recognizer.java b/src/main/mondrian/rolap/aggmatcher/Recognizer.java index b46b69067d..e2a3c4a1fb 100644 --- a/src/main/mondrian/rolap/aggmatcher/Recognizer.java +++ b/src/main/mondrian/rolap/aggmatcher/Recognizer.java @@ -5,10 +5,9 @@ // You must accept the terms of that agreement to use this software. // // Copyright (C) 2005-2005 Julian Hyde -// Copyright (C) 2005-2012 Pentaho and others +// Copyright (C) 2005-2014 Pentaho and others // All Rights Reserved. */ - package mondrian.rolap.aggmatcher; import mondrian.olap.*; @@ -757,6 +756,8 @@ protected void checkUnusedColumns() { dbFactTable.getName(), aggColumn.getName()); unusedColumnMsgs.put(aggColumn.getName(), msg); + // since the column has no usage it will be ignored + makeIgnore(aggColumn); } } for (String msg : unusedColumnMsgs.values()) { diff --git a/testsrc/main/mondrian/rolap/TestAggregationManager.java b/testsrc/main/mondrian/rolap/TestAggregationManager.java index 316b354ece..70b12428e4 100644 --- a/testsrc/main/mondrian/rolap/TestAggregationManager.java +++ b/testsrc/main/mondrian/rolap/TestAggregationManager.java @@ -5,16 +5,16 @@ // You must accept the terms of that agreement to use this software. // // Copyright (C) 2002-2005 Julian Hyde -// Copyright (C) 2005-2013 Pentaho and others +// Copyright (C) 2005-2014 Pentaho and others // All Rights Reserved. // // jhyde, 28 September, 2002 */ - package mondrian.rolap; import mondrian.olap.*; import mondrian.rolap.agg.*; +import mondrian.rolap.aggmatcher.AggStar; import mondrian.server.*; import mondrian.spi.Dialect; import mondrian.test.SqlPattern; @@ -24,6 +24,10 @@ import java.util.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + /** * Unit test for {@link AggregationManager}. * @@ -2689,6 +2693,282 @@ public void testMondrian1271() { false, false, true); } + public void testAggStarWithIgnoredColumnsRequiresRollup() { + propSaver.set(propSaver.properties.GenerateFormattedSql, true); + propSaver.set(propSaver.properties.ReadAggregates, true); + propSaver.set(propSaver.properties.UseAggregates, true); + final TestContext context = + TestContext.instance().withSchema( + "" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + "\n" + + ""); + RolapStar star = context.getConnection().getSchemaReader() + .getSchema().getStar("sales_fact_1997"); + AggStar aggStarSpy = spy( + getAggStar(star, "agg_c_10_sales_fact_1997")); + // make sure the test AggStar will be prioritized first + when(aggStarSpy.getSize()).thenReturn(0); + context.getConnection().getSchemaReader() + .getSchema().getStar("sales_fact_1997").addAggStar(aggStarSpy); + boolean[] rollup = { false }; + AggStar returnedStar = AggregationManager + .findAgg( + star, aggStarSpy.getLevelBitKey(), + aggStarSpy.getMeasureBitKey(), rollup); + assertTrue( + "Rollup should be true since AggStar has ignored columns ", + rollup[0]); + assertEquals(aggStarSpy, returnedStar); + assertTrue( + "Columns marked with AggIgnoreColumn, so AggStar " + + ".hasIgnoredColumns() should be true", + aggStarSpy.hasIgnoredColumns()); + String sqlMysql = + "select\n" + + " `agg_c_10_sales_fact_1997`.`the_year` as `c0`,\n" + + " sum(`agg_c_10_sales_fact_1997`.`unit_sales`) as `m0`\n" + + "from\n" + + " `agg_c_10_sales_fact_1997` as `agg_c_10_sales_fact_1997`\n" + + "where\n" + + " `agg_c_10_sales_fact_1997`.`the_year` = 1997\n" + + "group by\n" + + " `agg_c_10_sales_fact_1997`.`the_year`"; + String sqlOra = + "select\n" + + " \"agg_c_10_sales_fact_1997\".\"the_year\" as \"c0\",\n" + + " sum(\"agg_c_10_sales_fact_1997\".\"unit_sales\") as \"m0\"\n" + + "from\n" + + " \"agg_c_10_sales_fact_1997\" \"agg_c_10_sales_fact_1997\"\n" + + "where\n" + + " \"agg_c_10_sales_fact_1997\".\"the_year\" = 1997\n" + + "group by\n" + + " \"agg_c_10_sales_fact_1997\".\"the_year\""; + assertQuerySqlOrNot( + context, + "select Time.[1997] on 0 from sales", + new SqlPattern[]{ + new SqlPattern( + Dialect.DatabaseProduct.MYSQL, + sqlMysql, + sqlMysql.length()), + new SqlPattern( + Dialect.DatabaseProduct.ORACLE, + sqlOra, + sqlOra.length())}, + false, false, true); + } + + public void testAggStarWithUnusedColumnsRequiresRollup() { + propSaver.set(propSaver.properties.ReadAggregates, true); + propSaver.set(propSaver.properties.UseAggregates, true); + propSaver.set(propSaver.properties.GenerateFormattedSql, true); + final TestContext context = + TestContext.instance().withSchema( + "" + + "\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + ""); + RolapStar star = context.getConnection().getSchemaReader() + .getSchema().getStar("sales_fact_1997"); + AggStar aggStarSpy = spy( + getAggStar(star, "agg_c_special_sales_fact_1997")); + // make sure the test AggStar will be prioritized first + when(aggStarSpy.getSize()).thenReturn(0); + context.getConnection().getSchemaReader() + .getSchema().getStar("sales_fact_1997").addAggStar(aggStarSpy); + + boolean[] rollup = { false }; + AggStar returnedStar = AggregationManager + .findAgg( + star, aggStarSpy.getLevelBitKey(), + aggStarSpy.getMeasureBitKey(), rollup); + assertTrue( + "Rollup should be true since AggStar has ignored columns ", + rollup[0]); + assertEquals(aggStarSpy, returnedStar); + assertTrue( + "Unused columns are present, should be marked as " + + "having ignored columns.", aggStarSpy.hasIgnoredColumns()); + + String sqlOra = + "select\n" + + " \"customer\".\"gender\" as \"c0\",\n" + + " sum(\"agg_c_special_sales_fact_1997\".\"unit_sales_sum\") as \"m0\"\n" + + "from\n" + + " \"customer\" \"customer\",\n" + + " \"agg_c_special_sales_fact_1997\" \"agg_c_special_sales_fact_1997\"\n" + + "where\n" + + " \"agg_c_special_sales_fact_1997\".\"customer_id\" = \"customer\".\"customer_id\"\n" + + "group by\n" + + " \"customer\".\"gender\""; + String sqlMysql = + "select\n" + + " `customer`.`gender` as `c0`,\n" + + " sum(`agg_c_special_sales_fact_1997`.`unit_sales_sum`) as `m0`\n" + + "from\n" + + " `customer` as `customer`,\n" + + " `agg_c_special_sales_fact_1997` as `agg_c_special_sales_fact_1997`\n" + + "where\n" + + " `agg_c_special_sales_fact_1997`.`customer_id` = `customer`.`customer_id`\n" + + "group by\n" + + " `customer`.`gender`"; + assertQuerySqlOrNot( + context, + "select gender.gender.members on 0 from sales", + new SqlPattern[]{ + new SqlPattern( + Dialect.DatabaseProduct.MYSQL, + sqlMysql, + sqlMysql.length()), + new SqlPattern( + Dialect.DatabaseProduct.ORACLE, + sqlOra, + sqlOra.length())}, + false, false, true); + } + + + public void testAggStarWithIgnoredColumnsAndCountDistinct() { + propSaver.set(propSaver.properties.ReadAggregates, true); + propSaver.set(propSaver.properties.UseAggregates, true); + propSaver.set(propSaver.properties.GenerateFormattedSql, true); + final TestContext context = + TestContext.instance().withSchema( + "" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "\n" + + ""); + RolapStar star = context.getConnection().getSchemaReader() + .getSchema().getStar("sales_fact_1997"); + AggStar aggStarSpy = spy( + getAggStar(star, "agg_g_ms_pcat_sales_fact_1997")); + // make sure the test AggStar will be prioritized first + when(aggStarSpy.getSize()).thenReturn(0); + context.getConnection().getSchemaReader() + .getSchema().getStar("sales_fact_1997").addAggStar(aggStarSpy); + boolean[] rollup = { false }; + AggStar returnedStar = AggregationManager + .findAgg( + star, aggStarSpy.getLevelBitKey(), + aggStarSpy.getMeasureBitKey(), rollup); + assertNull( + "Should not find an agg star given that ignored or unused " + + "columns are present, and loading distinct count measure", + returnedStar); + String sqlOra = + "select\n" + + " \"time_by_day\".\"the_year\" as \"c0\",\n" + + " count(distinct \"sales_fact_1997\".\"customer_id\") as \"m0\"\n" + + "from\n" + + " \"time_by_day\" \"time_by_day\",\n" + + " \"sales_fact_1997\" \"sales_fact_1997\"\n" + + "where\n" + + " \"sales_fact_1997\".\"time_id\" = \"time_by_day\".\"time_id\"\n" + + "and\n" + + " \"time_by_day\".\"the_year\" = 1997\n" + + "group by\n" + + " \"time_by_day\".\"the_year\""; + String sqlMysql = + "select\n" + + " `time_by_day`.`the_year` as `c0`,\n" + + " count(distinct `sales_fact_1997`.`customer_id`) as `m0`\n" + + "from\n" + + " `time_by_day` as `time_by_day`,\n" + + " `sales_fact_1997` as `sales_fact_1997`\n" + + "where\n" + + " `sales_fact_1997`.`time_id` = `time_by_day`.`time_id`\n" + + "and\n" + + " `time_by_day`.`the_year` = 1997\n" + + "group by\n" + + " `time_by_day`.`the_year`"; + assertQuerySqlOrNot( + context, + "select Time.[1997] on 0 from sales where " + + "measures.[Customer Count]", + new SqlPattern[]{ + new SqlPattern( + Dialect.DatabaseProduct.MYSQL, + sqlMysql, + sqlMysql.length()), + new SqlPattern( + Dialect.DatabaseProduct.ORACLE, + sqlOra, + sqlOra.length())}, + false, false, true); + } + + + private AggStar getAggStar(RolapStar star, String aggStarName) { + for (AggStar aggStar : star.getAggStars()) { + if (aggStar.getFactTable().getName().equals(aggStarName)) { + return aggStar; + } + } + return null; + } + } // End TestAggregationManager.java diff --git a/testsrc/main/mondrian/rolap/agg/AggregationOnDistinctCountMeasuresTest.java b/testsrc/main/mondrian/rolap/agg/AggregationOnDistinctCountMeasuresTest.java index bd00e23426..ed1170a2c9 100644 --- a/testsrc/main/mondrian/rolap/agg/AggregationOnDistinctCountMeasuresTest.java +++ b/testsrc/main/mondrian/rolap/agg/AggregationOnDistinctCountMeasuresTest.java @@ -1,12 +1,11 @@ /* -* This software is subject to the terms of the Eclipse Public License v1.0 -* Agreement, available at the following URL: -* http://www.eclipse.org/legal/epl-v10.html. -* You must accept the terms of that agreement to use this software. -* -* Copyright (c) 2002-2013 Pentaho Corporation.. All rights reserved. +// This software is subject to the terms of the Eclipse Public License v1.0 +// Agreement, available at the following URL: +// http://www.eclipse.org/legal/epl-v10.html. +// You must accept the terms of that agreement to use this software. +// +// Copyright (c) 2002-2014 Pentaho Corporation.. All rights reserved. */ - package mondrian.rolap.agg; import mondrian.calc.TupleList; @@ -1112,9 +1111,6 @@ public void testOptimizeListWhenTuplesAreFormedWithDifferentLevels() { // directly into optimizeChildren like some of the tests below rather // than using SQL pattern verification. SqlPattern[] patterns = { - /* - new SqlPattern(SqlPattern.Dialect.DERBY, derbySql, derbySql), - */ new SqlPattern( Dialect.DatabaseProduct.ACCESS, accessSql, accessSql)}; @@ -1550,9 +1546,9 @@ public void testDistinctCountAggMeasure() { + " " + " " + " " - + " " - + " " - + " " + + " " + + " " + + " " + " " + " " + " " @@ -1560,6 +1556,12 @@ public void testDistinctCountAggMeasure() { + " " + " " + " " + + "\n" + + " \n" + + " " + " " + ""; final String query =