From 1adcd1c52e87a2f59d9006a29fb9476174968a60 Mon Sep 17 00:00:00 2001 From: Jonathan Ellis Date: Tue, 19 Apr 2011 17:13:19 +0000 Subject: [PATCH] merge from 0.8 git-svn-id: https://svn.apache.org/repos/asf/cassandra/trunk@1095139 13f79535-47bb-0310-9956-ffa450edef68 --- .rat-excludes | 3 +- CHANGES.txt | 6 +- NEWS.txt | 31 ++-- build.xml | 2 +- conf/cassandra.yaml | 2 +- .../hadoop/pig/CassandraStorage.java | 41 +++--- debian/cassandra.install | 1 - debian/changelog | 6 + doc/cql/CQL.html | 23 ++- .../cql/jdbc/CassandraPreparedStatement.java | 21 +++ .../cql/jdbc/CassandraResultSet.java | 55 +++++-- .../cassandra/cql/jdbc/ColumnDecoder.java | 2 +- .../cassandra/cql/jdbc/TypedColumn.java | 6 +- .../apache/cassandra/cql/JdbcDriverTest.java | 5 +- .../cql/jdbc/PreparedStatementTest.java | 38 ++++- drivers/py/cql/cassandra/constants.py | 2 +- drivers/py/cql/cassandra/ttypes.py | 22 ++- drivers/py/cql/cursor.py | 8 +- drivers/txpy/txcql/cassandra/constants.py | 2 +- drivers/txpy/txcql/cassandra/ttypes.py | 22 ++- interface/cassandra.thrift | 12 +- .../org/apache/cassandra/thrift/Column.java | 64 ++++----- .../apache/cassandra/thrift/Constants.java | 2 +- .../org/apache/cassandra/thrift/KsDef.java | 115 ++++++++++++++- .../cache/ConcurrentLinkedHashCache.java | 21 +++ .../ConcurrentLinkedHashCacheProvider.java | 21 +++ .../cassandra/cache/FreeableMemory.java | 21 +++ .../org/apache/cassandra/cache/ICache.java | 21 +++ .../cassandra/cache/IRowCacheProvider.java | 21 +++ .../cassandra/cache/SerializingCache.java | 21 +++ .../cache/SerializingCacheProvider.java | 21 +++ .../org/apache/cassandra/cli/CliClient.java | 2 +- .../apache/cassandra/config/CFMetaData.java | 4 +- .../cassandra/config/DatabaseDescriptor.java | 2 +- .../apache/cassandra/config/KSMetaData.java | 19 ++- src/java/org/apache/cassandra/cql/Cql.g | 2 - .../apache/cassandra/cql/QueryProcessor.java | 135 +++++++++++------- .../cassandra/db/ColumnFamilyStore.java | 10 +- .../cassandra/db/HintedHandOffManager.java | 4 +- .../apache/cassandra/db/MeteredFlusher.java | 21 +++ .../hadoop/ColumnFamilyRecordWriter.java | 2 +- .../apache/cassandra/io/CompactionType.java | 21 +++ .../cassandra/io/ICompactSerializer3.java | 21 +++ .../cassandra/io/sstable/SSTableReader.java | 6 +- .../cassandra/io/util/MemoryInputStream.java | 23 ++- .../cassandra/io/util/MemoryOutputStream.java | 21 +++ .../cassandra/net/OutboundTcpConnection.java | 4 +- .../cassandra/service/AntiEntropyService.java | 4 + .../cassandra/thrift/CassandraServer.java | 26 +--- .../cassandra/thrift/ThriftValidation.java | 23 ++- .../org/apache/cassandra/cli/CliHelp.yaml | 19 ++- .../cassandra/db/MeteredFlusherTest.java | 21 +++ test/system/__init__.py | 6 +- test/system/test_cql.py | 30 ++-- test/system/test_thrift_server.py | 16 +-- .../cassandra/cache/CacheProviderTest.java | 21 +++ .../cassandra/client/TestRingCache.java | 2 +- .../cassandra/db/ColumnFamilyStoreTest.java | 29 ++++ .../apache/cassandra/db/SystemTableTest.java | 21 +++ .../cassandra/db/marshal/UUIDTypeTest.java | 21 +++ .../service/EmbeddedCassandraServiceTest.java | 6 +- .../org/apache/cassandra/stress/Session.java | 4 + .../stress/operations/IndexedRangeSlicer.java | 4 +- .../cassandra/stress/util/Operation.java | 24 +++- 64 files changed, 967 insertions(+), 245 deletions(-) diff --git a/.rat-excludes b/.rat-excludes index 0559dedf100d..953d3c06c4c7 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -2,7 +2,8 @@ debian/** .classpath .rat-excludes -.project +**/.project +**/.pydevproject CHANGES.txt .git/** **/*.json diff --git a/CHANGES.txt b/CHANGES.txt index 52a6284b34c0..036e055dc07a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,10 +1,10 @@ -0.8-dev +0.8.0-beta1 * remove Avro RPC support (CASSANDRA-926) * adds support for columns that act as incr/decr counters (CASSANDRA-1072, 1937, 1944, 1936, 2101, 2093, 2288, 2105, 2384, 2236, 2342, 2454) * CQL (CASSANDRA-1703, 1704, 1705, 1706, 1707, 1708, 1710, 1711, 1940, - 2124, 2302, 2277) + 2124, 2302, 2277, 2493) * avoid double RowMutation serialization on write path (CASSANDRA-1800) * make NetworkTopologyStrategy the default (CASSANDRA-1960) * configurable internode encryption (CASSANDRA-1567, 2152) @@ -55,6 +55,8 @@ * Try harder to close files after compaction (CASSANDRA-2431) * re-set bootstrapped flag after move finishes (CASSANDRA-2435) * use 64KB flush buffer instead of in_memory_compaction_limit (CASSANDRA-2463) + * fix duplicate results from CFS.scan (CASSANDRA-2406) + * avoid caching token-only decoratedkeys (CASSANDRA-2416) 0.7.4 diff --git a/NEWS.txt b/NEWS.txt index 5ec79e189146..bb20e6db2096 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -3,15 +3,29 @@ Upgrading --------- - Avro record classes used in map/reduce and Hadoop streaming code have - moved from org.apache.cassandra.avro to org.apache.cassandra.hadoop.avro, - applications using these classes will need to be updated accordingly. + - Upgrading from version 0.7.1 or later can be done with a rolling + restart, one node at a time. You do not need to bring down the + whole cluster. + - Avro record classes used in map/reduce and Hadoop streaming code have + moved from org.apache.cassandra.avro to org.apache.cassandra.hadoop.avro, + applications using these classes will need to be updated accordingly. + - The loadbalance command has been removed from nodetool. For similar + behavior, decommission then rebootstrap with empty initial_token. + - repair now works on a token range, rather than the entire ring. This + means that "run repair against each node" will now repair the ring with + no redundant work. - Upgrading from version 0.7.1 or later can be done with a rolling restart, - one node at a time. You do not need to bring down the whole cluster. - - The loadbalance command has been removed from nodetool. For similar - behavior, decommission then rebootstrap with empty initial_token. +Features +-------- + - added CQL client API and JDBC/DBAPI2-compliant drivers for Java and + Python, respectively (see: drivers/ subdirectory and doc/cql) + - added distributed Counters feature; + see http://wiki.apache.org/cassandra/Counters + - optional intranode encryption; see comments around 'encryption_options' + in cassandra.yaml + - compaction multithreading and rate-limiting; see + 'compaction_multithreading' and 'compaction_throughput_mb_per_sec' in + cassandra.yaml Other ----- @@ -55,7 +69,6 @@ Upgrading data files before rebuilding, just in case. ->>>>>>> .merge-right.r1073884 0.7.1 ===== diff --git a/build.xml b/build.xml index 506eedabb9e1..1ee7bba351bb 100644 --- a/build.xml +++ b/build.xml @@ -55,7 +55,7 @@ - + diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 436667bbfe10..7d817c1f993c 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -252,7 +252,7 @@ in_memory_compaction_limit_in_mb: 64 # Enables multiple compactions to execute at once. This is highly recommended # for preserving read performance in a mixed read/write workload as this # avoids sstables from accumulating during long running compactions. -compaction_multithreading: true +compaction_multithreading: false # Throttles compaction to the given total throughput across the entire # system. The faster you insert data, the faster you need to compact in diff --git a/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java b/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java index d5906f82aa83..ca41b8d3d6ba 100644 --- a/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java +++ b/contrib/pig/src/java/org/apache/cassandra/hadoop/pig/CassandraStorage.java @@ -56,11 +56,11 @@ import org.apache.thrift.transport.TTransportException; /** - * A LoadFunc wrapping ColumnFamilyInputFormat. + * A LoadStoreFunc for retrieving data from and storing data to Cassandra * * A row from a standard CF will be returned as nested tuples: (key, ((name1, val1), (name2, val2))). */ -public class CassandraStorage extends LoadFunc implements StoreFuncInterface, LoadPushDown +public class CassandraStorage extends LoadFunc implements StoreFuncInterface { // system environment variables that can be set to configure connection info: // alternatively, Hadoop JobConf variables can be set using keys from ConfigHelper @@ -68,7 +68,7 @@ public class CassandraStorage extends LoadFunc implements StoreFuncInterface, Lo public final static String PIG_INITIAL_ADDRESS = "PIG_INITIAL_ADDRESS"; public final static String PIG_PARTITIONER = "PIG_PARTITIONER"; - private static String UDFCONTEXT_SCHEMA_KEY = "cassandra.schema"; + private static String UDFCONTEXT_SCHEMA_KEY_PREFIX = "cassandra.schema"; private final static ByteBuffer BOUND = ByteBufferUtil.EMPTY_BYTE_BUFFER; private static final Log logger = LogFactory.getLog(CassandraStorage.class); @@ -169,7 +169,7 @@ private CfDef getCfDef() { UDFContext context = UDFContext.getUDFContext(); Properties property = context.getUDFProperties(CassandraStorage.class); - return cfdefFromString(property.getProperty(UDFCONTEXT_SCHEMA_KEY)); + return cfdefFromString(property.getProperty(getSchemaContextKey())); } private List getDefaultMarshallers(CfDef cfDef) throws IOException @@ -226,7 +226,7 @@ public void prepareToRead(RecordReader reader, PigSplit split) this.reader = reader; } - private void setLocationFromUri(String location) throws IOException + private void setLocationFromUri(String location) throws IOException { // parse uri into keyspace and columnfamily String names[]; @@ -396,7 +396,7 @@ public void putNext(Tuple t) throws ExecException, IOException if (validators.get(column.name) == null) // Have to special case BytesType to convert DataByteArray into ByteBuffer if (marshallers.get(1) instanceof BytesType) - column.value = ByteBuffer.wrap(((DataByteArray) pair.get(1)).get()); + column.value = objToBB(pair.get(1)); else column.value = marshallers.get(1).decompose(pair.get(1)); else @@ -428,27 +428,16 @@ public void cleanupOnFailure(String failure, Job job) { } - /* LoadPushDown methods */ - - public List getFeatures() { - return Arrays.asList(LoadPushDown.OperatorSet.PROJECTION); - } - - public RequiredFieldResponse pushProjection(RequiredFieldList requiredFieldList) throws FrontendException - { - return new RequiredFieldResponse(true); - } - - /* Methods to get the column family schema from Cassandra */ private void initSchema() { UDFContext context = UDFContext.getUDFContext(); Properties property = context.getUDFProperties(CassandraStorage.class); - + + String schemaContextKey = getSchemaContextKey(); // Only get the schema if we haven't already gotten it - if (!property.containsKey(UDFCONTEXT_SCHEMA_KEY)) + if (!property.containsKey(schemaContextKey)) { Cassandra.Client client = null; try @@ -466,7 +455,7 @@ private void initSchema() break; } } - property.setProperty(UDFCONTEXT_SCHEMA_KEY, cfdefToString(cfDef)); + property.setProperty(schemaContextKey, cfdefToString(cfDef)); } catch (TException e) { @@ -532,4 +521,14 @@ private static CfDef cfdefFromString(String st) } return cfDef; } + + private String getSchemaContextKey() + { + StringBuilder sb = new StringBuilder(UDFCONTEXT_SCHEMA_KEY_PREFIX); + sb.append('.'); + sb.append(keyspace); + sb.append('.'); + sb.append(column_family); + return sb.toString(); + } } diff --git a/debian/cassandra.install b/debian/cassandra.install index 0f4ae0f106fc..977deb23f559 100644 --- a/debian/cassandra.install +++ b/debian/cassandra.install @@ -10,6 +10,5 @@ bin/clustertool usr/bin bin/json2sstable usr/bin bin/sstable2json usr/bin bin/sstablekeys usr/bin -bin/schematool usr/bin lib/*.jar usr/share/cassandra lib/licenses usr/share/doc/cassandra diff --git a/debian/changelog b/debian/changelog index f2df6be92372..9fe0c7edbafd 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +cassandra (0.8.0~beta1) unstable; urgency=low + + * New beta release. + + -- Eric Evans Mon, 18 Apr 2011 11:41:09 -0500 + cassandra (0.7.4) unstable; urgency=low * New stable point release. diff --git a/doc/cql/CQL.html b/doc/cql/CQL.html index 21a808181bf7..a29227929a29 100644 --- a/doc/cql/CQL.html +++ b/doc/cql/CQL.html @@ -1,3 +1,24 @@ + +

Cassandra Query Language (CQL) v1.0.0

Table of Contents

  1. Cassandra Query Language (CQL) v1.0.0
    1. Table of Contents
    2. USE
    3. SELECT
      1. Specifying Columns
      2. Column Family
      3. Consistency Level
      4. Filtering rows
      5. Limits
    4. UPDATE
      1. Column Family
      2. Consistency Level
      3. Specifying Columns and Row
    5. DELETE
      1. Specifying Columns
      2. Column Family
      3. Consistency Level
      4. Specifying Rows
    6. TRUNCATE
    7. CREATE KEYSPACE
    8. CREATE COLUMNFAMILY
      1. Specifying Key Type
      2. Specifying Column Type (optional)
      3. Column Family Options (optional)
    9. CREATE INDEX
    10. DROP
    11. Common Idioms
      1. Specifying Consistency
      2. Term specification
  2. Versioning
  3. Changes

USE

Synopsis:

USE <KEYSPACE>;

A USE statement consists of the USE keyword, followed by a valid keyspace name. Its purpose is to assign the per-connection, current working keyspace. All subsequent keyspace-specific actions will be performed in the context of the supplied value.

SELECT

Synopsis:

SELECT [FIRST N] [REVERSED] <SELECT EXPR> FROM <COLUMN FAMILY> [USING <CONSISTENCY>]
         [WHERE <CLAUSE>] [LIMIT N];
@@ -39,4 +60,4 @@

DROP statements result in the immediate, irreversible removal of keyspace and column family namespaces.

Common Idioms

Specifying Consistency

... USING <CONSISTENCY> ...

Consistency level specifications are made up the keyword USING, followed by a consistency level identifier. Valid consistency levels are as follows:

  • CONSISTENCY ZERO
  • CONSISTENCY ONE (default)
  • CONSISTENCY QUORUM
  • CONSISTENCY ALL
  • CONSISTENCY DCQUORUM
  • CONSISTENCY DCQUORUMSYNC

Term specification

Terms are used in statements to specify things such as keyspaces, column families, indexes, column names and values, and keyword arguments. The rules governing term specification are as follows:

  • Any single quoted string literal (example: 'apple').
  • Unquoted alpha-numeric strings that begin with a letter (example: carrot).
  • Unquoted numeric literals (example: 100).
  • UUID strings in hyphen-delimited hex notation (example: 1438fc5c-4ff6-11e0-b97f-0026c650d722).

Terms which do not conform to these rules result in an exception.

How column name/value terms are interpreted is determined by the configured type.

typeterm
asciiAny string which can be decoded using ASCII charset
text / varcharAny string which can be decoded using UTF8 charset
uuidStandard UUID string format (hyphen-delimited hex notation)
uuidStandard UUID string format (hyphen-delimited hex notation)
uuidThe string now, to represent a type-1 (time-based) UUID with a date-time component based on the current time
uuidNumeric value representing milliseconds since epoch
uuidAn iso8601 timestamp
bigintNumeric value capable of fitting in 8 bytes
varintNumeric value of arbitrary size
byteaHex-encoded strings (converted directly to the corresponding bytes)

Versioning

Versioning of the CQL language adheres to the Semantic Versioning guidelines. Versions take the form X.Y.Z where X, Y, and Z are integer values representing major, minor, and patch level respectively. There is no correlation between Cassandra release versions and the CQL language version.

versiondescription
PatchThe patch version is incremented when bugs are fixed.
MinorMinor version increments occur when new, but backward compatible, functionality is introduced.
MajorThe major version must be bumped when backward incompatible changes are introduced. This should rarely (if ever) occur.

Changes

Tue, 22 Mar 2011 18:10:28 -0700 - Eric Evans <eevans@rackspace.com>
  * Initial version, 1.0.0
-
\ No newline at end of file + diff --git a/drivers/java/src/org/apache/cassandra/cql/jdbc/CassandraPreparedStatement.java b/drivers/java/src/org/apache/cassandra/cql/jdbc/CassandraPreparedStatement.java index 04be1ee99c77..fa70cada5cc7 100644 --- a/drivers/java/src/org/apache/cassandra/cql/jdbc/CassandraPreparedStatement.java +++ b/drivers/java/src/org/apache/cassandra/cql/jdbc/CassandraPreparedStatement.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cql.jdbc; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import org.apache.cassandra.db.marshal.*; diff --git a/drivers/java/src/org/apache/cassandra/cql/jdbc/CassandraResultSet.java b/drivers/java/src/org/apache/cassandra/cql/jdbc/CassandraResultSet.java index 341f60521a09..6e94c6704706 100644 --- a/drivers/java/src/org/apache/cassandra/cql/jdbc/CassandraResultSet.java +++ b/drivers/java/src/org/apache/cassandra/cql/jdbc/CassandraResultSet.java @@ -75,11 +75,13 @@ class CassandraResultSet implements ResultSet private List values = new ArrayList(); /** The value map. */ + // TODO should map so we can throw appropriate exception if user asks for non-existant column name private Map valueMap = new WeakHashMap(); private final RsMetaData meta; private final AbstractType nameType; + private boolean wasNull; /** * Instantiates a new cassandra result set. @@ -356,7 +358,11 @@ public byte getByte(String arg0) throws SQLException */ public byte[] getBytes(int index) throws SQLException { - return values.get(index-1) != null ? ((ByteBuffer)values.get(index-1).getValue()).array() : null; + TypedColumn column = values.get(index - 1); + assert column != null; + Object value = column.getValue(); + wasNull = value == null; + return value == null ? null : ((ByteBuffer) value).array(); } /** @@ -367,7 +373,9 @@ public byte[] getBytes(int index) throws SQLException public byte[] getBytes(String name) throws SQLException { String nameAsString = decoder.colNameAsString(keyspace, columnFamily, name); - return valueMap.get(nameAsString) != null ? ((ByteBuffer)valueMap.get(nameAsString)).array() : null; + Object value = valueMap.get(nameAsString); + wasNull = value == null; + return value == null ? null : ((ByteBuffer) value).array(); } /** @@ -544,7 +552,11 @@ public int getHoldability() throws SQLException */ public int getInt(int index) throws SQLException { - return values.get(index-1) != null ? ((BigInteger)values.get(index-1).getValue()).intValue() : null; + TypedColumn column = values.get(index - 1); + assert column != null; + Object value = column.getValue(); + wasNull = value == null; + return value == null ? 0 : ((BigInteger) value).intValue(); } /** @@ -555,7 +567,9 @@ public int getInt(int index) throws SQLException public int getInt(String name) throws SQLException { String nameAsString = decoder.colNameAsString(keyspace, columnFamily, name); - return valueMap.get(nameAsString) != null ? ((BigInteger)valueMap.get(nameAsString)).intValue() : null; + Object value = valueMap.get(nameAsString); + wasNull = value == null; + return value == null ? 0 : ((BigInteger) value).intValue(); } /** @@ -565,7 +579,12 @@ public int getInt(String name) throws SQLException */ public long getLong(int index) throws SQLException { - return values.get(index-1) != null ? (Long)values.get(index-1).getValue() : null; + assert values != null; + TypedColumn column = values.get(index - 1); + assert column != null; + Object value = column.getValue(); + wasNull = value == null; + return value == null ? 0 : (Long) value; } /** @@ -576,7 +595,9 @@ public long getLong(int index) throws SQLException public long getLong(String name) throws SQLException { String nameAsString = decoder.colNameAsString(keyspace, columnFamily, name); - return valueMap.get(nameAsString) != null ? (Long)valueMap.get(nameAsString) : null; + Object value = valueMap.get(nameAsString); + wasNull = value == null; + return value == null ? 0 : (Long) value; } /** @@ -655,7 +676,11 @@ public String getNString(String arg0) throws SQLException */ public Object getObject(int index) throws SQLException { - return values.get(index-1) == null ? null : values.get(index-1).getValue(); + TypedColumn column = values.get(index - 1); + assert column != null; + Object value = column.getValue(); + wasNull = value == null; + return value; } /** @@ -666,7 +691,9 @@ public Object getObject(int index) throws SQLException public Object getObject(String name) throws SQLException { String nameAsString = decoder.colNameAsString(keyspace, columnFamily, name); - return valueMap.get(nameAsString); + Object value = valueMap.get(nameAsString); + wasNull = value == null; + return value; } /** @@ -796,7 +823,11 @@ public Statement getStatement() throws SQLException */ public String getString(int index) throws SQLException { - return values.get(index-1) != null ? ColumnDecoder.colValueAsString(values.get(index-1).getValue()) : null; + TypedColumn column = values.get(index - 1); + assert column != null; + Object value = column.getValue(); + wasNull = value == null; + return value == null ? null : ColumnDecoder.colValueAsString(value); } /** @@ -807,7 +838,9 @@ public String getString(int index) throws SQLException public String getString(String name) throws SQLException { String nameAsString = this.decoder.colNameAsString(this.keyspace, this.columnFamily, name); - return valueMap.get(nameAsString) != null ? ColumnDecoder.colValueAsString(valueMap.get(nameAsString)) : null; + Object value = valueMap.get(nameAsString); + wasNull = value == null; + return value == null ? null : ColumnDecoder.colValueAsString(value); } /** @@ -1999,7 +2032,7 @@ public void updateTimestamp(String arg0, Timestamp arg1) throws SQLException */ public boolean wasNull() throws SQLException { - throw new UnsupportedOperationException("method not supported"); + return wasNull; } /** diff --git a/drivers/java/src/org/apache/cassandra/cql/jdbc/ColumnDecoder.java b/drivers/java/src/org/apache/cassandra/cql/jdbc/ColumnDecoder.java index cf8c56eb4351..6feb189a7d92 100644 --- a/drivers/java/src/org/apache/cassandra/cql/jdbc/ColumnDecoder.java +++ b/drivers/java/src/org/apache/cassandra/cql/jdbc/ColumnDecoder.java @@ -181,7 +181,7 @@ public static String colValueAsString(Object value) { else if (value instanceof byte[]) return ByteBufferUtil.bytesToHex(ByteBuffer.wrap((byte[])value)); else - return value.toString(); + return value == null ? null : value.toString(); } /** constructs a typed column */ diff --git a/drivers/java/src/org/apache/cassandra/cql/jdbc/TypedColumn.java b/drivers/java/src/org/apache/cassandra/cql/jdbc/TypedColumn.java index 4b9b82162574..ef31182a4a19 100644 --- a/drivers/java/src/org/apache/cassandra/cql/jdbc/TypedColumn.java +++ b/drivers/java/src/org/apache/cassandra/cql/jdbc/TypedColumn.java @@ -39,11 +39,11 @@ class TypedColumn public TypedColumn(AbstractType comparator, byte[] name, AbstractType validator, byte[] value) { ByteBuffer bbName = ByteBuffer.wrap(name); - ByteBuffer bbValue = ByteBuffer.wrap(value); + ByteBuffer bbValue = value == null ? null : ByteBuffer.wrap(value); this.name = comparator.compose(bbName); - this.value = validator.compose(bbValue); + this.value = value == null ? null : validator.compose(bbValue); nameString = comparator.getString(bbName); - valueString = validator.getString(bbValue); + valueString = value == null ? null : validator.getString(bbValue); this.validator = validator; } diff --git a/drivers/java/test/org/apache/cassandra/cql/JdbcDriverTest.java b/drivers/java/test/org/apache/cassandra/cql/JdbcDriverTest.java index e3cd52917f5b..5c997d16f6be 100644 --- a/drivers/java/test/org/apache/cassandra/cql/JdbcDriverTest.java +++ b/drivers/java/test/org/apache/cassandra/cql/JdbcDriverTest.java @@ -398,8 +398,11 @@ public void testWithDeleteStatement() throws SQLException { executeNoResults(con, statements[3*i]); ResultSet rs = executePreparedStatementWithResults(con, statements[3*i+1]); - assert !rs.next() : statements[3*i+1]; + rs.next(); + rs.getObject(1); + assert rs.wasNull(); rs.close(); + rs = executePreparedStatementWithResults(con, statements[3*i+2]); assert rs.next() : statements[3*i+2]; } diff --git a/drivers/java/test/org/apache/cassandra/cql/jdbc/PreparedStatementTest.java b/drivers/java/test/org/apache/cassandra/cql/jdbc/PreparedStatementTest.java index 9dd9c6187b7e..506cb681eccd 100644 --- a/drivers/java/test/org/apache/cassandra/cql/jdbc/PreparedStatementTest.java +++ b/drivers/java/test/org/apache/cassandra/cql/jdbc/PreparedStatementTest.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cql.jdbc; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import org.apache.cassandra.cql.EmbeddedServiceBase; import org.apache.cassandra.utils.FBUtilities; @@ -80,7 +101,8 @@ public void testBytes() throws SQLException stmt.setBytes(2, FBUtilities.toByteArray(i+100)); stmt.setBytes(3, key); ResultSet rs = stmt.executeQuery(); - assert !rs.next(); + rs.next(); + assert rs.getString(1) == null; assert rs.getString(2) == null; rs.close(); } } @@ -139,7 +161,8 @@ public void testUtf8() throws SQLException stmt.setString(2, "2\u6543\u3435\u6554"); stmt.setBytes(3, key); ResultSet rs = stmt.executeQuery(); - assert !rs.next(); + rs.next(); + assert rs.getString(1) == null; assert rs.getString(2) == null; rs.close(); } } @@ -198,7 +221,8 @@ public void testAscii() throws SQLException stmt.setString(2, "2"); stmt.setBytes(3, key); ResultSet rs = stmt.executeQuery(); - assert !rs.next(); + rs.next(); + assert rs.getString(1) == null; assert rs.getString(2) == null; rs.close(); } } @@ -257,7 +281,9 @@ public void testLong() throws SQLException stmt.setLong(2, 2); stmt.setBytes(3, key); ResultSet rs = stmt.executeQuery(); - assert !rs.next(); + rs.next(); + rs.getLong(1); + assert rs.wasNull(); rs.close(); } } @@ -316,7 +342,9 @@ public void testInteger() throws SQLException stmt.setInt(2, 2); stmt.setBytes(3, key); ResultSet rs = stmt.executeQuery(); - assert !rs.next(); + rs.next(); + rs.getInt(1); + assert rs.wasNull(); rs.close(); } } diff --git a/drivers/py/cql/cassandra/constants.py b/drivers/py/cql/cassandra/constants.py index 2502504d4f0e..ab365a6ddfbf 100644 --- a/drivers/py/cql/cassandra/constants.py +++ b/drivers/py/cql/cassandra/constants.py @@ -7,4 +7,4 @@ from thrift.Thrift import * from ttypes import * -VERSION = "20.1.0" +VERSION = "19.10.0" diff --git a/drivers/py/cql/cassandra/ttypes.py b/drivers/py/cql/cassandra/ttypes.py index bfdc26c39f58..cc69f46b9355 100644 --- a/drivers/py/cql/cassandra/ttypes.py +++ b/drivers/py/cql/cassandra/ttypes.py @@ -246,10 +246,6 @@ def write(self, oprot): def validate(self): if self.name is None: raise TProtocol.TProtocolException(message='Required field name is unset!') - if self.value is None: - raise TProtocol.TProtocolException(message='Required field value is unset!') - if self.timestamp is None: - raise TProtocol.TProtocolException(message='Required field timestamp is unset!') return @@ -2666,6 +2662,7 @@ class KsDef: - name - strategy_class - strategy_options + - replication_factor: deprecated - cf_defs """ @@ -2674,13 +2671,15 @@ class KsDef: (1, TType.STRING, 'name', None, None, ), # 1 (2, TType.STRING, 'strategy_class', None, None, ), # 2 (3, TType.MAP, 'strategy_options', (TType.STRING,None,TType.STRING,None), None, ), # 3 - (4, TType.LIST, 'cf_defs', (TType.STRUCT,(CfDef, CfDef.thrift_spec)), None, ), # 4 + (4, TType.I32, 'replication_factor', None, None, ), # 4 + (5, TType.LIST, 'cf_defs', (TType.STRUCT,(CfDef, CfDef.thrift_spec)), None, ), # 5 ) - def __init__(self, name=None, strategy_class=None, strategy_options=None, cf_defs=None,): + def __init__(self, name=None, strategy_class=None, strategy_options=None, replication_factor=None, cf_defs=None,): self.name = name self.strategy_class = strategy_class self.strategy_options = strategy_options + self.replication_factor = replication_factor self.cf_defs = cf_defs def read(self, iprot): @@ -2714,6 +2713,11 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 4: + if ftype == TType.I32: + self.replication_factor = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 5: if ftype == TType.LIST: self.cf_defs = [] (_etype68, _size65) = iprot.readListBegin() @@ -2750,8 +2754,12 @@ def write(self, oprot): oprot.writeString(viter72) oprot.writeMapEnd() oprot.writeFieldEnd() + if self.replication_factor != None: + oprot.writeFieldBegin('replication_factor', TType.I32, 4) + oprot.writeI32(self.replication_factor) + oprot.writeFieldEnd() if self.cf_defs != None: - oprot.writeFieldBegin('cf_defs', TType.LIST, 4) + oprot.writeFieldBegin('cf_defs', TType.LIST, 5) oprot.writeListBegin(TType.STRUCT, len(self.cf_defs)) for iter73 in self.cf_defs: iter73.write(oprot) diff --git a/drivers/py/cql/cursor.py b/drivers/py/cql/cursor.py index c83e8afc3828..ec7884ff1056 100644 --- a/drivers/py/cql/cursor.py +++ b/drivers/py/cql/cursor.py @@ -22,8 +22,12 @@ from cql.marshal import prepare from cql.decoders import SchemaDecoder from cql.results import ResultSet -from cql.cassandra.ttypes import (Compression, CqlResultType, InvalidRequestException, - TApplicationException) +from cql.cassandra.ttypes import ( + Compression, + CqlResultType, + InvalidRequestException, + TApplicationException, + SchemaDisagreementException) class Cursor: diff --git a/drivers/txpy/txcql/cassandra/constants.py b/drivers/txpy/txcql/cassandra/constants.py index 2502504d4f0e..ab365a6ddfbf 100644 --- a/drivers/txpy/txcql/cassandra/constants.py +++ b/drivers/txpy/txcql/cassandra/constants.py @@ -7,4 +7,4 @@ from thrift.Thrift import * from ttypes import * -VERSION = "20.1.0" +VERSION = "19.10.0" diff --git a/drivers/txpy/txcql/cassandra/ttypes.py b/drivers/txpy/txcql/cassandra/ttypes.py index bfdc26c39f58..cc69f46b9355 100644 --- a/drivers/txpy/txcql/cassandra/ttypes.py +++ b/drivers/txpy/txcql/cassandra/ttypes.py @@ -246,10 +246,6 @@ def write(self, oprot): def validate(self): if self.name is None: raise TProtocol.TProtocolException(message='Required field name is unset!') - if self.value is None: - raise TProtocol.TProtocolException(message='Required field value is unset!') - if self.timestamp is None: - raise TProtocol.TProtocolException(message='Required field timestamp is unset!') return @@ -2666,6 +2662,7 @@ class KsDef: - name - strategy_class - strategy_options + - replication_factor: deprecated - cf_defs """ @@ -2674,13 +2671,15 @@ class KsDef: (1, TType.STRING, 'name', None, None, ), # 1 (2, TType.STRING, 'strategy_class', None, None, ), # 2 (3, TType.MAP, 'strategy_options', (TType.STRING,None,TType.STRING,None), None, ), # 3 - (4, TType.LIST, 'cf_defs', (TType.STRUCT,(CfDef, CfDef.thrift_spec)), None, ), # 4 + (4, TType.I32, 'replication_factor', None, None, ), # 4 + (5, TType.LIST, 'cf_defs', (TType.STRUCT,(CfDef, CfDef.thrift_spec)), None, ), # 5 ) - def __init__(self, name=None, strategy_class=None, strategy_options=None, cf_defs=None,): + def __init__(self, name=None, strategy_class=None, strategy_options=None, replication_factor=None, cf_defs=None,): self.name = name self.strategy_class = strategy_class self.strategy_options = strategy_options + self.replication_factor = replication_factor self.cf_defs = cf_defs def read(self, iprot): @@ -2714,6 +2713,11 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 4: + if ftype == TType.I32: + self.replication_factor = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 5: if ftype == TType.LIST: self.cf_defs = [] (_etype68, _size65) = iprot.readListBegin() @@ -2750,8 +2754,12 @@ def write(self, oprot): oprot.writeString(viter72) oprot.writeMapEnd() oprot.writeFieldEnd() + if self.replication_factor != None: + oprot.writeFieldBegin('replication_factor', TType.I32, 4) + oprot.writeI32(self.replication_factor) + oprot.writeFieldEnd() if self.cf_defs != None: - oprot.writeFieldBegin('cf_defs', TType.LIST, 4) + oprot.writeFieldBegin('cf_defs', TType.LIST, 5) oprot.writeListBegin(TType.STRUCT, len(self.cf_defs)) for iter73 in self.cf_defs: iter73.write(oprot) diff --git a/interface/cassandra.thrift b/interface/cassandra.thrift index 03cd18730df0..f59ca600e2aa 100644 --- a/interface/cassandra.thrift +++ b/interface/cassandra.thrift @@ -46,7 +46,7 @@ namespace rb CassandraThrift # for every edit that doesn't result in a change to major/minor. # # See the Semantic Versioning Specification (SemVer) http://semver.org. -const string VERSION = "20.1.0" +const string VERSION = "19.10.0" # @@ -61,8 +61,8 @@ const string VERSION = "20.1.0" */ struct Column { 1: required binary name, - 2: required binary value, - 3: required i64 timestamp, + 2: optional binary value, + 3: optional i64 timestamp, 4: optional i32 ttl, } @@ -402,7 +402,11 @@ struct KsDef { 1: required string name, 2: required string strategy_class, 3: optional map strategy_options, - 4: required list cf_defs, + + /** @deprecated */ + 4: optional i32 replication_factor, + + 5: required list cf_defs, } /** CQL query compression */ diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java index 6fa86facda7e..0a27d336b959 100644 --- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java +++ b/interface/thrift/gen-java/org/apache/cassandra/thrift/Column.java @@ -139,9 +139,9 @@ public String getFieldName() { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.REQUIRED, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); - tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.REQUIRED, + tmpMap.put(_Fields.VALUE, new org.apache.thrift.meta_data.FieldMetaData("value", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true))); - tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.REQUIRED, + tmpMap.put(_Fields.TIMESTAMP, new org.apache.thrift.meta_data.FieldMetaData("timestamp", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64))); tmpMap.put(_Fields.TTL, new org.apache.thrift.meta_data.FieldMetaData("ttl", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); @@ -153,15 +153,10 @@ public Column() { } public Column( - ByteBuffer name, - ByteBuffer value, - long timestamp) + ByteBuffer name) { this(); this.name = name; - this.value = value; - this.timestamp = timestamp; - setTimestampIsSet(true); } /** @@ -415,8 +410,8 @@ public boolean equals(Column that) { return false; } - boolean this_present_timestamp = true; - boolean that_present_timestamp = true; + boolean this_present_timestamp = true && this.isSetTimestamp(); + boolean that_present_timestamp = true && that.isSetTimestamp(); if (this_present_timestamp || that_present_timestamp) { if (!(this_present_timestamp && that_present_timestamp)) return false; @@ -450,7 +445,7 @@ public int hashCode() { if (present_value) builder.append(value); - boolean present_timestamp = true; + boolean present_timestamp = true && (isSetTimestamp()); builder.append(present_timestamp); if (present_timestamp) builder.append(timestamp); @@ -566,9 +561,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.t iprot.readStructEnd(); // check for required fields of primitive type, which can't be checked in the validate method - if (!isSetTimestamp()) { - throw new org.apache.thrift.protocol.TProtocolException("Required field 'timestamp' was not found in serialized data! Struct: " + toString()); - } validate(); } @@ -582,13 +574,17 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. oprot.writeFieldEnd(); } if (this.value != null) { - oprot.writeFieldBegin(VALUE_FIELD_DESC); - oprot.writeBinary(this.value); + if (isSetValue()) { + oprot.writeFieldBegin(VALUE_FIELD_DESC); + oprot.writeBinary(this.value); + oprot.writeFieldEnd(); + } + } + if (isSetTimestamp()) { + oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); + oprot.writeI64(this.timestamp); oprot.writeFieldEnd(); } - oprot.writeFieldBegin(TIMESTAMP_FIELD_DESC); - oprot.writeI64(this.timestamp); - oprot.writeFieldEnd(); if (isSetTtl()) { oprot.writeFieldBegin(TTL_FIELD_DESC); oprot.writeI32(this.ttl); @@ -610,18 +606,22 @@ public String toString() { org.apache.thrift.TBaseHelper.toString(this.name, sb); } first = false; - if (!first) sb.append(", "); - sb.append("value:"); - if (this.value == null) { - sb.append("null"); - } else { - org.apache.thrift.TBaseHelper.toString(this.value, sb); + if (isSetValue()) { + if (!first) sb.append(", "); + sb.append("value:"); + if (this.value == null) { + sb.append("null"); + } else { + org.apache.thrift.TBaseHelper.toString(this.value, sb); + } + first = false; + } + if (isSetTimestamp()) { + if (!first) sb.append(", "); + sb.append("timestamp:"); + sb.append(this.timestamp); + first = false; } - first = false; - if (!first) sb.append(", "); - sb.append("timestamp:"); - sb.append(this.timestamp); - first = false; if (isSetTtl()) { if (!first) sb.append(", "); sb.append("ttl:"); @@ -637,10 +637,6 @@ public void validate() throws org.apache.thrift.TException { if (name == null) { throw new org.apache.thrift.protocol.TProtocolException("Required field 'name' was not present! Struct: " + toString()); } - if (value == null) { - throw new org.apache.thrift.protocol.TProtocolException("Required field 'value' was not present! Struct: " + toString()); - } - // alas, we cannot check 'timestamp' because it's a primitive and you chose the non-beans generator. } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java index 32602fba5092..e589e2e3448a 100644 --- a/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java +++ b/interface/thrift/gen-java/org/apache/cassandra/thrift/Constants.java @@ -44,6 +44,6 @@ public class Constants { - public static final String VERSION = "20.1.0"; + public static final String VERSION = "19.10.0"; } diff --git a/interface/thrift/gen-java/org/apache/cassandra/thrift/KsDef.java b/interface/thrift/gen-java/org/apache/cassandra/thrift/KsDef.java index f276a2b195b4..f70dfdf55289 100644 --- a/interface/thrift/gen-java/org/apache/cassandra/thrift/KsDef.java +++ b/interface/thrift/gen-java/org/apache/cassandra/thrift/KsDef.java @@ -48,11 +48,16 @@ public class KsDef implements org.apache.thrift.TBase, jav private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); private static final org.apache.thrift.protocol.TField STRATEGY_CLASS_FIELD_DESC = new org.apache.thrift.protocol.TField("strategy_class", org.apache.thrift.protocol.TType.STRING, (short)2); private static final org.apache.thrift.protocol.TField STRATEGY_OPTIONS_FIELD_DESC = new org.apache.thrift.protocol.TField("strategy_options", org.apache.thrift.protocol.TType.MAP, (short)3); - private static final org.apache.thrift.protocol.TField CF_DEFS_FIELD_DESC = new org.apache.thrift.protocol.TField("cf_defs", org.apache.thrift.protocol.TType.LIST, (short)4); + private static final org.apache.thrift.protocol.TField REPLICATION_FACTOR_FIELD_DESC = new org.apache.thrift.protocol.TField("replication_factor", org.apache.thrift.protocol.TType.I32, (short)4); + private static final org.apache.thrift.protocol.TField CF_DEFS_FIELD_DESC = new org.apache.thrift.protocol.TField("cf_defs", org.apache.thrift.protocol.TType.LIST, (short)5); public String name; public String strategy_class; public Map strategy_options; + /** + * @deprecated + */ + public int replication_factor; public List cf_defs; /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ @@ -60,7 +65,11 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { NAME((short)1, "name"), STRATEGY_CLASS((short)2, "strategy_class"), STRATEGY_OPTIONS((short)3, "strategy_options"), - CF_DEFS((short)4, "cf_defs"); + /** + * @deprecated + */ + REPLICATION_FACTOR((short)4, "replication_factor"), + CF_DEFS((short)5, "cf_defs"); private static final Map byName = new HashMap(); @@ -81,7 +90,9 @@ public static _Fields findByThriftId(int fieldId) { return STRATEGY_CLASS; case 3: // STRATEGY_OPTIONS return STRATEGY_OPTIONS; - case 4: // CF_DEFS + case 4: // REPLICATION_FACTOR + return REPLICATION_FACTOR; + case 5: // CF_DEFS return CF_DEFS; default: return null; @@ -123,6 +134,8 @@ public String getFieldName() { } // isset id assignments + private static final int __REPLICATION_FACTOR_ISSET_ID = 0; + private BitSet __isset_bit_vector = new BitSet(1); public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { @@ -135,6 +148,8 @@ public String getFieldName() { new org.apache.thrift.meta_data.MapMetaData(org.apache.thrift.protocol.TType.MAP, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING), new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)))); + tmpMap.put(_Fields.REPLICATION_FACTOR, new org.apache.thrift.meta_data.FieldMetaData("replication_factor", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); tmpMap.put(_Fields.CF_DEFS, new org.apache.thrift.meta_data.FieldMetaData("cf_defs", org.apache.thrift.TFieldRequirementType.REQUIRED, new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, CfDef.class)))); @@ -160,6 +175,8 @@ public KsDef( * Performs a deep copy on other. */ public KsDef(KsDef other) { + __isset_bit_vector.clear(); + __isset_bit_vector.or(other.__isset_bit_vector); if (other.isSetName()) { this.name = other.name; } @@ -181,6 +198,7 @@ public KsDef(KsDef other) { } this.strategy_options = __this__strategy_options; } + this.replication_factor = other.replication_factor; if (other.isSetCf_defs()) { List __this__cf_defs = new ArrayList(); for (CfDef other_element : other.cf_defs) { @@ -199,6 +217,8 @@ public void clear() { this.name = null; this.strategy_class = null; this.strategy_options = null; + setReplication_factorIsSet(false); + this.replication_factor = 0; this.cf_defs = null; } @@ -285,6 +305,35 @@ public void setStrategy_optionsIsSet(boolean value) { } } + /** + * @deprecated + */ + public int getReplication_factor() { + return this.replication_factor; + } + + /** + * @deprecated + */ + public KsDef setReplication_factor(int replication_factor) { + this.replication_factor = replication_factor; + setReplication_factorIsSet(true); + return this; + } + + public void unsetReplication_factor() { + __isset_bit_vector.clear(__REPLICATION_FACTOR_ISSET_ID); + } + + /** Returns true if field replication_factor is set (has been assigned a value) and false otherwise */ + public boolean isSetReplication_factor() { + return __isset_bit_vector.get(__REPLICATION_FACTOR_ISSET_ID); + } + + public void setReplication_factorIsSet(boolean value) { + __isset_bit_vector.set(__REPLICATION_FACTOR_ISSET_ID, value); + } + public int getCf_defsSize() { return (this.cf_defs == null) ? 0 : this.cf_defs.size(); } @@ -350,6 +399,14 @@ public void setFieldValue(_Fields field, Object value) { } break; + case REPLICATION_FACTOR: + if (value == null) { + unsetReplication_factor(); + } else { + setReplication_factor((Integer)value); + } + break; + case CF_DEFS: if (value == null) { unsetCf_defs(); @@ -372,6 +429,9 @@ public Object getFieldValue(_Fields field) { case STRATEGY_OPTIONS: return getStrategy_options(); + case REPLICATION_FACTOR: + return new Integer(getReplication_factor()); + case CF_DEFS: return getCf_defs(); @@ -392,6 +452,8 @@ public boolean isSet(_Fields field) { return isSetStrategy_class(); case STRATEGY_OPTIONS: return isSetStrategy_options(); + case REPLICATION_FACTOR: + return isSetReplication_factor(); case CF_DEFS: return isSetCf_defs(); } @@ -438,6 +500,15 @@ public boolean equals(KsDef that) { return false; } + boolean this_present_replication_factor = true && this.isSetReplication_factor(); + boolean that_present_replication_factor = true && that.isSetReplication_factor(); + if (this_present_replication_factor || that_present_replication_factor) { + if (!(this_present_replication_factor && that_present_replication_factor)) + return false; + if (this.replication_factor != that.replication_factor) + return false; + } + boolean this_present_cf_defs = true && this.isSetCf_defs(); boolean that_present_cf_defs = true && that.isSetCf_defs(); if (this_present_cf_defs || that_present_cf_defs) { @@ -469,6 +540,11 @@ public int hashCode() { if (present_strategy_options) builder.append(strategy_options); + boolean present_replication_factor = true && (isSetReplication_factor()); + builder.append(present_replication_factor); + if (present_replication_factor) + builder.append(replication_factor); + boolean present_cf_defs = true && (isSetCf_defs()); builder.append(present_cf_defs); if (present_cf_defs) @@ -515,6 +591,16 @@ public int compareTo(KsDef other) { return lastComparison; } } + lastComparison = Boolean.valueOf(isSetReplication_factor()).compareTo(typedOther.isSetReplication_factor()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetReplication_factor()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.replication_factor, typedOther.replication_factor); + if (lastComparison != 0) { + return lastComparison; + } + } lastComparison = Boolean.valueOf(isSetCf_defs()).compareTo(typedOther.isSetCf_defs()); if (lastComparison != 0) { return lastComparison; @@ -575,7 +661,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.t org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); } break; - case 4: // CF_DEFS + case 4: // REPLICATION_FACTOR + if (field.type == org.apache.thrift.protocol.TType.I32) { + this.replication_factor = iprot.readI32(); + setReplication_factorIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + case 5: // CF_DEFS if (field.type == org.apache.thrift.protocol.TType.LIST) { { org.apache.thrift.protocol.TList _list37 = iprot.readListBegin(); @@ -633,6 +727,11 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. oprot.writeFieldEnd(); } } + if (isSetReplication_factor()) { + oprot.writeFieldBegin(REPLICATION_FACTOR_FIELD_DESC); + oprot.writeI32(this.replication_factor); + oprot.writeFieldEnd(); + } if (this.cf_defs != null) { oprot.writeFieldBegin(CF_DEFS_FIELD_DESC); { @@ -679,6 +778,12 @@ public String toString() { } first = false; } + if (isSetReplication_factor()) { + if (!first) sb.append(", "); + sb.append("replication_factor:"); + sb.append(this.replication_factor); + first = false; + } if (!first) sb.append(", "); sb.append("cf_defs:"); if (this.cf_defs == null) { @@ -714,6 +819,8 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bit_vector = new BitSet(1); read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); diff --git a/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCache.java b/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCache.java index b10d3194bb17..8691647fc50e 100644 --- a/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCache.java +++ b/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCache.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cache; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.util.Collection; import java.util.Map; diff --git a/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCacheProvider.java b/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCacheProvider.java index 40999ce339ca..00a01728595a 100644 --- a/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCacheProvider.java +++ b/src/java/org/apache/cassandra/cache/ConcurrentLinkedHashCacheProvider.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cache; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap; import org.apache.cassandra.db.ColumnFamily; diff --git a/src/java/org/apache/cassandra/cache/FreeableMemory.java b/src/java/org/apache/cassandra/cache/FreeableMemory.java index 1e015d68e3c4..d662075597bc 100644 --- a/src/java/org/apache/cassandra/cache/FreeableMemory.java +++ b/src/java/org/apache/cassandra/cache/FreeableMemory.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cache; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.io.IOException; diff --git a/src/java/org/apache/cassandra/cache/ICache.java b/src/java/org/apache/cassandra/cache/ICache.java index 9f988c71edcf..0eebdb4d3627 100644 --- a/src/java/org/apache/cassandra/cache/ICache.java +++ b/src/java/org/apache/cassandra/cache/ICache.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cache; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.util.Set; diff --git a/src/java/org/apache/cassandra/cache/IRowCacheProvider.java b/src/java/org/apache/cassandra/cache/IRowCacheProvider.java index 08616871720f..c1310ddea4e5 100644 --- a/src/java/org/apache/cassandra/cache/IRowCacheProvider.java +++ b/src/java/org/apache/cassandra/cache/IRowCacheProvider.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cache; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.DecoratedKey; diff --git a/src/java/org/apache/cassandra/cache/SerializingCache.java b/src/java/org/apache/cassandra/cache/SerializingCache.java index 6874b8bbbece..4171152d3091 100644 --- a/src/java/org/apache/cassandra/cache/SerializingCache.java +++ b/src/java/org/apache/cassandra/cache/SerializingCache.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cache; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.io.DataOutputStream; import java.io.IOError; diff --git a/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java b/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java index 77ebeea55b2e..a80c4b453ffd 100644 --- a/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java +++ b/src/java/org/apache/cassandra/cache/SerializingCacheProvider.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cache; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import org.apache.cassandra.config.ConfigurationException; import org.apache.cassandra.db.ColumnFamily; diff --git a/src/java/org/apache/cassandra/cli/CliClient.java b/src/java/org/apache/cassandra/cli/CliClient.java index 45b110b43b85..06810e63227a 100644 --- a/src/java/org/apache/cassandra/cli/CliClient.java +++ b/src/java/org/apache/cassandra/cli/CliClient.java @@ -802,7 +802,7 @@ else if (columnSpecCnt == 1) if(superColumnName != null) parent.setSuper_column(superColumnName); - Column columnToInsert = new Column(columnName, columnValueInBytes, FBUtilities.timestampMicros()); + Column columnToInsert = new Column(columnName).setValue(columnValueInBytes).setTimestamp(FBUtilities.timestampMicros()); // children count = 3 mean that we have ttl in arguments if (statement.getChildCount() == 3) diff --git a/src/java/org/apache/cassandra/config/CFMetaData.java b/src/java/org/apache/cassandra/config/CFMetaData.java index bff93d9e7607..e35348a0b7db 100644 --- a/src/java/org/apache/cassandra/config/CFMetaData.java +++ b/src/java/org/apache/cassandra/config/CFMetaData.java @@ -254,7 +254,9 @@ private static CFMetaData newSystemMetadata(String cfName, int cfId, String comm .gcGraceSeconds(0) .memSize(memtableThroughPutInMB) .memOps(sizeMemtableOperations(memtableThroughPutInMB)) - .mergeShardsChance(0.0); + .mergeShardsChance(0.0) + .rowCacheSavePeriod(0) + .keyCacheSavePeriod(0); } public static CFMetaData newIndexMetadata(CFMetaData parent, ColumnDefinition info, AbstractType columnComparator) diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 8a6167b2d6b6..e4f74b31ab03 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -86,7 +86,7 @@ public class DatabaseDescriptor private static RequestSchedulerOptions requestSchedulerOptions; public static final UUID INITIAL_VERSION = new UUID(4096, 0); // has type nibble set to 1, everything else to zero. - private static UUID defsVersion = INITIAL_VERSION; + private static volatile UUID defsVersion = INITIAL_VERSION; /** * Inspect the classpath to find storage configuration file diff --git a/src/java/org/apache/cassandra/config/KSMetaData.java b/src/java/org/apache/cassandra/config/KSMetaData.java index d8e2777f50f1..cb6887869255 100644 --- a/src/java/org/apache/cassandra/config/KSMetaData.java +++ b/src/java/org/apache/cassandra/config/KSMetaData.java @@ -29,6 +29,8 @@ import org.apache.cassandra.io.SerDeUtils; import org.apache.cassandra.locator.AbstractReplicationStrategy; import org.apache.cassandra.locator.NetworkTopologyStrategy; +import org.apache.cassandra.thrift.KsDef; + import org.apache.commons.lang.StringUtils; public final class KSMetaData @@ -48,7 +50,22 @@ public KSMetaData(String name, Class stra cfmap.put(cfm.cfName, cfm); this.cfMetaData = Collections.unmodifiableMap(cfmap); } - + + public static Map backwardsCompatibleOptions(KsDef ks_def) + { + Map options; + if (ks_def.isSetReplication_factor()) + { + options = new HashMap(ks_def.strategy_options == null ? Collections.emptyMap() : ks_def.strategy_options); + options.put("replication_factor", String.valueOf(ks_def.replication_factor)); + } + else + { + options = ks_def.strategy_options; + } + return options; + } + public int hashCode() { return name.hashCode(); diff --git a/src/java/org/apache/cassandra/cql/Cql.g b/src/java/org/apache/cassandra/cql/Cql.g index 35890982122d..6d4c707f07df 100644 --- a/src/java/org/apache/cassandra/cql/Cql.g +++ b/src/java/org/apache/cassandra/cql/Cql.g @@ -382,11 +382,9 @@ K_FROM: F R O M; K_WHERE: W H E R E; K_AND: A N D; K_KEY: K E Y; -K_COLUMN: C O L (U M N)?; K_INSERT: I N S E R T; K_UPDATE: U P D A T E; K_WITH: W I T H; -K_ROW: R O W; K_LIMIT: L I M I T; K_USING: U S I N G; K_CONSISTENCY: C O N S I S T E N C Y; diff --git a/src/java/org/apache/cassandra/cql/QueryProcessor.java b/src/java/org/apache/cassandra/cql/QueryProcessor.java index 0bd2774a9b11..338b0a9d52a4 100644 --- a/src/java/org/apache/cassandra/cql/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql/QueryProcessor.java @@ -467,20 +467,22 @@ public static CqlResult process(String queryString, ClientState clientState) if (StatementType.requiresKeyspace.contains(statement.type)) keyspace = clientState.getKeyspace(); - CqlResult avroResult = new CqlResult(); + CqlResult result = new CqlResult(); logger.debug("CQL statement type: {}", statement.type.toString()); - + CFMetaData metadata; + AbstractType comparator; switch (statement.type) { case SELECT: SelectStatement select = (SelectStatement)statement.statement; clientState.hasColumnFamilyAccess(select.getColumnFamily(), Permission.READ); - validateColumnFamily(keyspace, select.getColumnFamily(), false); + metadata = validateColumnFamily(keyspace, select.getColumnFamily(), false); + comparator = metadata.getComparatorFor(null); validateSelect(keyspace, select); List rows = null; - + // By-key if (!select.isKeyRange() && (select.getKeys().size() > 0)) { @@ -489,12 +491,12 @@ public static CqlResult process(String queryString, ClientState clientState) // Only return the column count, (of the at-most 1 row). if (select.isCountOperation()) { - avroResult.type = CqlResultType.INT; + result.type = CqlResultType.INT; if (rows.size() > 0) - avroResult.setNum(rows.get(0).cf != null ? rows.get(0).cf.getSortedColumns().size() : 0); + result.setNum(rows.get(0).cf != null ? rows.get(0).cf.getSortedColumns().size() : 0); else - avroResult.setNum(0); - return avroResult; + result.setNum(0); + return result; } } else @@ -511,8 +513,8 @@ public static CqlResult process(String queryString, ClientState clientState) } } - List avroRows = new ArrayList(); - avroResult.type = CqlResultType.ROWS; + List cqlRows = new ArrayList(); + result.type = CqlResultType.ROWS; // Create the result set for (org.apache.cassandra.db.Row row : rows) @@ -520,38 +522,26 @@ public static CqlResult process(String queryString, ClientState clientState) /// No results for this row if (row.cf == null) continue; - - List avroColumns = new ArrayList(); - - for (IColumn column : row.cf.getSortedColumns()) - { - if (column.isMarkedForDelete()) - continue; - Column avroColumn = new Column(); - avroColumn.name = column.name(); - avroColumn.value = column.value(); - avroColumn.timestamp = column.timestamp(); - avroColumns.add(avroColumn); - } - + + List thriftColumns = extractThriftColumns(select, comparator, row); // Create a new row, add the columns to it, and then add it to the list of rows - CqlRow avroRow = new CqlRow(); - avroRow.key = row.key.key; - avroRow.columns = avroColumns; + CqlRow cqlRow = new CqlRow(); + cqlRow.key = row.key.key; + cqlRow.columns = thriftColumns; if (select.isColumnsReversed()) - Collections.reverse(avroRow.columns); - avroRows.add(avroRow); + Collections.reverse(cqlRow.columns); + cqlRows.add(cqlRow); } - avroResult.rows = avroRows; - return avroResult; + result.rows = cqlRows; + return result; case INSERT: // insert uses UpdateStatement case UPDATE: UpdateStatement update = (UpdateStatement)statement.statement; batchUpdate(clientState, Collections.singletonList(update), update.getConsistencyLevel()); - avroResult.type = CqlResultType.VOID; - return avroResult; + result.type = CqlResultType.VOID; + return result; case BATCH_UPDATE: BatchUpdateStatement batch = (BatchUpdateStatement)statement.statement; @@ -562,14 +552,14 @@ public static CqlResult process(String queryString, ClientState clientState) "Consistency level must be set on the BATCH, not individual UPDATE statements"); batchUpdate(clientState, batch.getUpdates(), batch.getConsistencyLevel()); - avroResult.type = CqlResultType.VOID; - return avroResult; + result.type = CqlResultType.VOID; + return result; case USE: clientState.setKeyspace((String)statement.statement); - avroResult.type = CqlResultType.VOID; + result.type = CqlResultType.VOID; - return avroResult; + return result; case TRUNCATE: String columnFamily = (String)statement.statement; @@ -588,14 +578,14 @@ public static CqlResult process(String queryString, ClientState clientState) throw (UnavailableException) new UnavailableException().initCause(e); } - avroResult.type = CqlResultType.VOID; - return avroResult; + result.type = CqlResultType.VOID; + return result; case DELETE: DeleteStatement delete = (DeleteStatement)statement.statement; clientState.hasColumnFamilyAccess(delete.getColumnFamily(), Permission.WRITE); - CFMetaData metadata = validateColumnFamily(keyspace, delete.getColumnFamily(), false); - AbstractType comparator = metadata.getComparatorFor(null); + metadata = validateColumnFamily(keyspace, delete.getColumnFamily(), false); + comparator = metadata.getComparatorFor(null); AbstractType keyType = DatabaseDescriptor.getCFMetaData(keyspace, delete.getColumnFamily()).getKeyValidator(); @@ -627,8 +617,8 @@ public static CqlResult process(String queryString, ClientState clientState) throw new TimedOutException(); } - avroResult.type = CqlResultType.VOID; - return avroResult; + result.type = CqlResultType.VOID; + return result; case CREATE_KEYSPACE: CreateKeyspaceStatement create = (CreateKeyspaceStatement)statement.statement; @@ -656,8 +646,8 @@ public static CqlResult process(String queryString, ClientState clientState) throw ex; } - avroResult.type = CqlResultType.VOID; - return avroResult; + result.type = CqlResultType.VOID; + return result; case CREATE_COLUMNFAMILY: CreateColumnFamilyStatement createCf = (CreateColumnFamilyStatement)statement.statement; @@ -681,8 +671,8 @@ public static CqlResult process(String queryString, ClientState clientState) throw ex; } - avroResult.type = CqlResultType.VOID; - return avroResult; + result.type = CqlResultType.VOID; + return result; case CREATE_INDEX: CreateIndexStatement createIdx = (CreateIndexStatement)statement.statement; @@ -737,8 +727,8 @@ public static CqlResult process(String queryString, ClientState clientState) throw ex; } - avroResult.type = CqlResultType.VOID; - return avroResult; + result.type = CqlResultType.VOID; + return result; case DROP_KEYSPACE: String deleteKeyspace = (String)statement.statement; @@ -762,8 +752,8 @@ public static CqlResult process(String queryString, ClientState clientState) throw ex; } - avroResult.type = CqlResultType.VOID; - return avroResult; + result.type = CqlResultType.VOID; + return result; case DROP_COLUMNFAMILY: String deleteColumnFamily = (String)statement.statement; @@ -787,14 +777,51 @@ public static CqlResult process(String queryString, ClientState clientState) throw ex; } - avroResult.type = CqlResultType.VOID; - return avroResult; + result.type = CqlResultType.VOID; + return result; } return null; // We should never get here. } - + + private static List extractThriftColumns(SelectStatement select, AbstractType comparator, Row row) + { + List thriftColumns = new ArrayList(); + if (select.isColumnRange()) + { + // preserve comparator order + for (IColumn c : row.cf.getSortedColumns()) + { + if (c.isMarkedForDelete()) + continue; + thriftColumns.add(new Column(c.name()).setValue(c.value()).setTimestamp(c.timestamp())); + } + } + else + { + // order columns in the order they were asked for + for (Term term : select.getColumnNames()) + { + ByteBuffer name; + try + { + name = term.getByteBuffer(comparator); + } + catch (InvalidRequestException e) + { + throw new AssertionError(e); + } + IColumn c = row.cf.getColumn(name); + if (c == null || c.isMarkedForDelete()) + thriftColumns.add(new Column().setName(name)); + else + thriftColumns.add(new Column(c.name()).setValue(c.value()).setTimestamp(c.timestamp())); + } + } + return thriftColumns; + } + private static CQLStatement getStatement(String queryStr) throws InvalidRequestException, RecognitionException { // Lexer and parser diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index cc2a707f48ea..ed8615b8a464 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -1459,6 +1459,10 @@ public List scan(IndexClause clause, AbstractBounds range, IFilter dataFilt ByteBuffer startKey = clause.start_key; QueryPath path = new QueryPath(columnFamily); + // we need to store last data key accessed to avoid duplicate results + // because in the while loop new iteration we can access the same column if start_key was not set + ByteBuffer lastDataKey = null; + // fetch row keys matching the primary expression, fetch the slice predicate for each // and filter by remaining expressions. repeat until finished w/ assigned range or index row is exhausted. outer: @@ -1490,10 +1494,11 @@ public List scan(IndexClause clause, AbstractBounds range, IFilter dataFilt continue; dataKey = column.name(); n++; + DecoratedKey dk = partitioner.decorateKey(dataKey); if (!range.right.equals(partitioner.getMinimumToken()) && range.right.compareTo(dk.token) < 0) break outer; - if (!range.contains(dk.token)) + if (!range.contains(dk.token) || dataKey.equals(lastDataKey)) continue; // get the row columns requested, and additional columns for the expressions if necessary @@ -1534,7 +1539,8 @@ public List scan(IndexClause clause, AbstractBounds range, IFilter dataFilt } if (n < clause.count || startKey.equals(dataKey)) break; - startKey = dataKey; + + lastDataKey = startKey = dataKey; } return rows; diff --git a/src/java/org/apache/cassandra/db/HintedHandOffManager.java b/src/java/org/apache/cassandra/db/HintedHandOffManager.java index 687bd2cc3b05..2c53e8217f60 100644 --- a/src/java/org/apache/cassandra/db/HintedHandOffManager.java +++ b/src/java/org/apache/cassandra/db/HintedHandOffManager.java @@ -279,13 +279,13 @@ private void deliverHintsToEndpoint(InetAddress endpoint) throws IOException, Di { try { - logger_.info("Checking remote schema before delivering hints"); + logger_.debug("Checking remote schema before delivering hints"); int waited = waitForSchemaAgreement(endpoint); // sleep a random amount to stagger handoff delivery from different replicas. // (if we had to wait, then gossiper randomness took care of that for us already.) if (waited == 0) { int sleep = new Random().nextInt(60000); - logger_.info("Sleeping {}ms to stagger hint delivery", sleep); + logger_.debug("Sleeping {}ms to stagger hint delivery", sleep); Thread.sleep(sleep); } if (!Gossiper.instance.getEndpointStateForEndpoint(endpoint).isAlive()) diff --git a/src/java/org/apache/cassandra/db/MeteredFlusher.java b/src/java/org/apache/cassandra/db/MeteredFlusher.java index 6abb5406b69e..d36faed63b87 100644 --- a/src/java/org/apache/cassandra/db/MeteredFlusher.java +++ b/src/java/org/apache/cassandra/db/MeteredFlusher.java @@ -1,4 +1,25 @@ package org.apache.cassandra.db; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.util.ArrayList; import java.util.Collections; diff --git a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java index 114e6c4070be..d1f245bee194 100644 --- a/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java +++ b/src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordWriter.java @@ -206,7 +206,7 @@ private SliceRange avroToThrift(org.apache.cassandra.hadoop.avro.SliceRange asr) private Column avroToThrift(org.apache.cassandra.hadoop.avro.Column acol) { - return new Column(acol.name, acol.value, acol.timestamp); + return new Column(acol.name).setValue(acol.value).setTimestamp(acol.timestamp); } /** diff --git a/src/java/org/apache/cassandra/io/CompactionType.java b/src/java/org/apache/cassandra/io/CompactionType.java index 5aefc86c021d..99366bb545f3 100644 --- a/src/java/org/apache/cassandra/io/CompactionType.java +++ b/src/java/org/apache/cassandra/io/CompactionType.java @@ -1,4 +1,25 @@ package org.apache.cassandra.io; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + public enum CompactionType { diff --git a/src/java/org/apache/cassandra/io/ICompactSerializer3.java b/src/java/org/apache/cassandra/io/ICompactSerializer3.java index 8b85fa184d90..136de40452cb 100644 --- a/src/java/org/apache/cassandra/io/ICompactSerializer3.java +++ b/src/java/org/apache/cassandra/io/ICompactSerializer3.java @@ -1,4 +1,25 @@ package org.apache.cassandra.io; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + public interface ICompactSerializer3 extends ICompactSerializer2 { diff --git a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java index dd2c4df9f18e..9d21a014317f 100644 --- a/src/java/org/apache/cassandra/io/sstable/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableReader.java @@ -500,8 +500,9 @@ public List> getPositionsForRanges(Collection ranges) public void cacheKey(DecoratedKey key, Long info) { + assert key.key != null; // avoid keeping a permanent reference to the original key buffer - DecoratedKey copiedKey = new DecoratedKey(key.token, key.key == null ? null : ByteBufferUtil.clone(key.key)); + DecoratedKey copiedKey = new DecoratedKey(key.token, ByteBufferUtil.clone(key.key)); keyCache.put(new Pair(descriptor, copiedKey), info); } @@ -570,7 +571,8 @@ public long getPosition(DecoratedKey decoratedKey, Operator op) if (op == Operator.EQ) bloomFilterTracker.addTruePositive(); // store exact match for the key - cacheKey(decoratedKey, dataPosition); + if (decoratedKey.key != null) + cacheKey(decoratedKey, dataPosition); } return dataPosition; } diff --git a/src/java/org/apache/cassandra/io/util/MemoryInputStream.java b/src/java/org/apache/cassandra/io/util/MemoryInputStream.java index 878ef445796a..6beb3eb1a8e6 100644 --- a/src/java/org/apache/cassandra/io/util/MemoryInputStream.java +++ b/src/java/org/apache/cassandra/io/util/MemoryInputStream.java @@ -1,4 +1,25 @@ package org.apache.cassandra.io.util; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.io.Closeable; import java.io.DataInput; @@ -42,4 +63,4 @@ public void close() { // do nothing. } -} \ No newline at end of file +} diff --git a/src/java/org/apache/cassandra/io/util/MemoryOutputStream.java b/src/java/org/apache/cassandra/io/util/MemoryOutputStream.java index 964aa9add663..235b9d7c4ca3 100644 --- a/src/java/org/apache/cassandra/io/util/MemoryOutputStream.java +++ b/src/java/org/apache/cassandra/io/util/MemoryOutputStream.java @@ -1,4 +1,25 @@ package org.apache.cassandra.io.util; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.io.OutputStream; diff --git a/src/java/org/apache/cassandra/net/OutboundTcpConnection.java b/src/java/org/apache/cassandra/net/OutboundTcpConnection.java index 8e911aff05fc..74a379a3743f 100644 --- a/src/java/org/apache/cassandra/net/OutboundTcpConnection.java +++ b/src/java/org/apache/cassandra/net/OutboundTcpConnection.java @@ -116,8 +116,8 @@ private void writeConnected(ByteBuffer bb) } catch (IOException e) { - logger.info("error writing to " + endpoint); - logger.debug("error was ", e); + if (logger.isDebugEnabled()) + logger.debug("error writing to " + endpoint, e); disconnect(); } } diff --git a/src/java/org/apache/cassandra/service/AntiEntropyService.java b/src/java/org/apache/cassandra/service/AntiEntropyService.java index b118a75ab17a..bcb4dd8f0e3c 100644 --- a/src/java/org/apache/cassandra/service/AntiEntropyService.java +++ b/src/java/org/apache/cassandra/service/AntiEntropyService.java @@ -286,6 +286,7 @@ public static class Validator implements Runnable private transient long validated; private transient MerkleTree.TreeRange range; private transient MerkleTree.TreeRangeIterator ranges; + private transient DecoratedKey lastKey; public final static MerkleTree.RowHash EMPTY_ROW = new MerkleTree.RowHash(null, new byte[0]); @@ -361,6 +362,9 @@ public void prepare(ColumnFamilyStore cfs) public void add(AbstractCompactedRow row) { assert request.range.contains(row.key.token) : row.key.token + " is not contained in " + request.range; + assert lastKey == null || lastKey.compareTo(row.key) < 0 + : "row " + row.key + " received out of order wrt " + lastKey; + lastKey = row.key; if (range == null) range = ranges.next(); diff --git a/src/java/org/apache/cassandra/thrift/CassandraServer.java b/src/java/org/apache/cassandra/thrift/CassandraServer.java index dd1e77e4df5f..8f6ee43870b0 100644 --- a/src/java/org/apache/cassandra/thrift/CassandraServer.java +++ b/src/java/org/apache/cassandra/thrift/CassandraServer.java @@ -24,7 +24,6 @@ import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.util.*; -import java.util.Map.Entry; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; @@ -55,7 +54,6 @@ import org.apache.cassandra.service.StorageProxy; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; import org.apache.thrift.TException; public class CassandraServer implements Cassandra.Iface @@ -146,7 +144,7 @@ public List thriftifySubColumns(Collection columns) { continue; } - Column thrift_column = new Column(column.name(), column.value(), column.timestamp()); + Column thrift_column = new Column(column.name()).setValue(column.value()).setTimestamp(column.timestamp()); if (column instanceof ExpiringColumn) { thrift_column.setTtl(((ExpiringColumn) column).getTimeToLive()); @@ -195,7 +193,7 @@ public List thriftifyColumns(Collection columns, b } else { - Column thrift_column = new Column(column.name(), column.value(), column.timestamp()); + Column thrift_column = new Column(column.name()).setValue(column.value()).setTimestamp(column.timestamp()); if (column instanceof ExpiringColumn) { thrift_column.setTtl(((ExpiringColumn) column).getTimeToLive()); @@ -858,21 +856,10 @@ public synchronized String system_add_keyspace(KsDef ks_def) cfDefs.add(CFMetaData.convertToCFMetaData(cfDef)); } - // Attempt to instantiate the ARS, which will throw a ConfigException if - // the strategy_options aren't fully formed or if the ARS Classname is invalid. - TokenMetadata tmd = StorageService.instance.getTokenMetadata(); - IEndpointSnitch eps = DatabaseDescriptor.getEndpointSnitch(); - Class cls = AbstractReplicationStrategy.getClass(ks_def.strategy_class); - AbstractReplicationStrategy strat = AbstractReplicationStrategy - .createReplicationStrategy(ks_def.name, - cls, - tmd, - eps, - ks_def.strategy_options); - + ThriftValidation.validateKsDef(ks_def); KSMetaData ksm = new KSMetaData(ks_def.name, AbstractReplicationStrategy.getClass(ks_def.strategy_class), - ks_def.strategy_options, + KSMetaData.backwardsCompatibleOptions(ks_def), cfDefs.toArray(new CFMetaData[cfDefs.size()])); applyMigrationOnStage(new AddKeyspace(ksm)); @@ -891,7 +878,7 @@ public synchronized String system_add_keyspace(KsDef ks_def) throw ex; } } - + public synchronized String system_drop_keyspace(String keyspace) throws InvalidRequestException, SchemaDisagreementException, TException { @@ -932,9 +919,10 @@ public synchronized String system_update_keyspace(KsDef ks_def) try { + ThriftValidation.validateKsDef(ks_def); KSMetaData ksm = new KSMetaData(ks_def.name, AbstractReplicationStrategy.getClass(ks_def.strategy_class), - ks_def.strategy_options); + KSMetaData.backwardsCompatibleOptions(ks_def)); applyMigrationOnStage(new UpdateKeyspace(ksm)); return DatabaseDescriptor.getDefsVersion().toString(); } diff --git a/src/java/org/apache/cassandra/thrift/ThriftValidation.java b/src/java/org/apache/cassandra/thrift/ThriftValidation.java index e6eda4832f39..19b9a4347e4d 100644 --- a/src/java/org/apache/cassandra/thrift/ThriftValidation.java +++ b/src/java/org/apache/cassandra/thrift/ThriftValidation.java @@ -23,16 +23,16 @@ import java.nio.ByteBuffer; import java.util.*; -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ConfigurationException; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.config.*; import org.apache.cassandra.db.*; import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.MarshalException; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -379,6 +379,10 @@ public static void validateSlicePredicate(CFMetaData metadata, ByteBuffer scName public static void validateColumnData(CFMetaData metadata, Column column) throws InvalidRequestException { validateTtl(column); + if (!column.isSetValue()) + throw new InvalidRequestException("Column value is required"); + if (!column.isSetTimestamp()) + throw new InvalidRequestException("Column timestamp is required"); try { AbstractType validator = metadata.getValueValidator(column.name); @@ -550,4 +554,15 @@ public static void validateCommutativeForWrite(CFMetaData metadata, ConsistencyL throw new InvalidRequestException("cannot achieve CL > CL.ONE without replicate_on_write on columnfamily " + metadata.cfName); } } + + static void validateKsDef(KsDef ks_def) throws ConfigurationException + { + // Attempt to instantiate the ARS, which will throw a ConfigException if + // the strategy_options aren't fully formed or if the ARS Classname is invalid. + Map options = KSMetaData.backwardsCompatibleOptions(ks_def); + TokenMetadata tmd = StorageService.instance.getTokenMetadata(); + IEndpointSnitch eps = DatabaseDescriptor.getEndpointSnitch(); + Class cls = AbstractReplicationStrategy.getClass(ks_def.strategy_class); + AbstractReplicationStrategy.createReplicationStrategy(ks_def.name, cls, tmd, eps, options); + } } diff --git a/src/resources/org/apache/cassandra/cli/CliHelp.yaml b/src/resources/org/apache/cassandra/cli/CliHelp.yaml index 2f8e38e7afaf..2b927f016218 100644 --- a/src/resources/org/apache/cassandra/cli/CliHelp.yaml +++ b/src/resources/org/apache/cassandra/cli/CliHelp.yaml @@ -1,3 +1,20 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Help file for online commands in Yaml. banner: | @@ -1147,4 +1164,4 @@ commands: - EACH_QUORUM - ANY - Note: Consistency level ANY can only be used for write operations. \ No newline at end of file + Note: Consistency level ANY can only be used for write operations. diff --git a/test/long/org/apache/cassandra/db/MeteredFlusherTest.java b/test/long/org/apache/cassandra/db/MeteredFlusherTest.java index 4e507d21bdeb..aa1fdd2640b3 100644 --- a/test/long/org/apache/cassandra/db/MeteredFlusherTest.java +++ b/test/long/org/apache/cassandra/db/MeteredFlusherTest.java @@ -1,4 +1,25 @@ package org.apache.cassandra.db; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.io.IOException; import java.nio.ByteBuffer; diff --git a/test/system/__init__.py b/test/system/__init__.py index f6b2d5746053..ab49198c81a1 100644 --- a/test/system/__init__.py +++ b/test/system/__init__.py @@ -33,7 +33,7 @@ def get_thrift_client(host='127.0.0.1', port=9170): socket = TSocket.TSocket(host, port) transport = TTransport.TFramedTransport(socket) - protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport) + protocol = TBinaryProtocol.TBinaryProtocol(transport) client = Cassandra.Client(protocol) client.transport = transport return client @@ -149,7 +149,7 @@ def close_client(self): def define_schema(self): keyspace1 = Cassandra.KsDef('Keyspace1', 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor':'1'}, - [ + cf_defs=[ Cassandra.CfDef('Keyspace1', 'Standard1'), Cassandra.CfDef('Keyspace1', 'Standard2'), Cassandra.CfDef('Keyspace1', 'StandardLong1', comparator_type='LongType'), @@ -168,7 +168,7 @@ def define_schema(self): ]) keyspace2 = Cassandra.KsDef('Keyspace2', 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor':'1'}, - [ + cf_defs=[ Cassandra.CfDef('Keyspace2', 'Standard1'), Cassandra.CfDef('Keyspace2', 'Standard3'), Cassandra.CfDef('Keyspace2', 'Super3', column_type='Super', subcomparator_type='BytesType'), diff --git a/test/system/test_cql.py b/test/system/test_cql.py index dfcb42d1f374..b716776b0fa6 100644 --- a/test/system/test_cql.py +++ b/test/system/test_cql.py @@ -136,13 +136,16 @@ def test_select_simple(self): def test_select_columns(self): "retrieve multiple columns" cursor = init() + # we deliberately request columns in non-comparator order cursor.execute(""" - SELECT 'cd1', 'col' FROM StandardString1 WHERE KEY = 'kd' + SELECT ca1, col, cd1 FROM StandardString1 WHERE KEY = 'kd' """) d = cursor.description - assert "cd1" in [col_dscptn[0] for col_dscptn in d] - assert "col" in [col_dscptn[0] for col_dscptn in d] + assert ['Row Key', 'ca1', 'col', 'cd1'] == [col_dscptn[0] for col_dscptn in d], d + row = cursor.fetchone() + # check that the column that didn't exist in the row comes back as null + assert ['kd', None, 'val', 'vd1'] == row, row def test_select_row_range(self): "retrieve a range of rows with columns" @@ -307,9 +310,8 @@ def test_delete_columns(self): cursor.execute(""" SELECT 'cd1', 'col' FROM StandardString1 WHERE KEY = 'kd' """) - colnames = [col_d[0] for col_d in cursor.description] - assert "cd1" in colnames - assert "col" in colnames + assert ['Row Key', 'cd1', 'col'] == [col_d[0] for col_d in cursor.description] + cursor.execute(""" DELETE 'cd1', 'col' FROM StandardString1 WHERE KEY = 'kd' """) @@ -317,7 +319,7 @@ def test_delete_columns(self): SELECT 'cd1', 'col' FROM StandardString1 WHERE KEY = 'kd' """) r = cursor.fetchone() - assert len(r) == 1 + assert ['kd', None, None] == r, r def test_delete_columns_multi_rows(self): "delete columns from multiple rows" @@ -325,22 +327,22 @@ def test_delete_columns_multi_rows(self): cursor.execute("SELECT 'col' FROM StandardString1 WHERE KEY = 'kc'") r = cursor.fetchone() - assert len(r) == 2 + assert ['kc', 'val'] == r, r cursor.execute("SELECT 'col' FROM StandardString1 WHERE KEY = 'kd'") r = cursor.fetchone() - assert len(r) == 2 + assert ['kd', 'val'] == r, r cursor.execute(""" DELETE 'col' FROM StandardString1 WHERE KEY IN ('kc', 'kd') """) cursor.execute("SELECT 'col' FROM StandardString1 WHERE KEY = 'kc'") r = cursor.fetchone() - assert len(r) == 1 + assert ['kc', None] == r, r cursor.execute("SELECT 'col' FROM StandardString1 WHERE KEY = 'kd'") r = cursor.fetchone() - assert len(r) == 1 + assert ['kd', None] == r, r def test_delete_rows(self): "delete entire rows" @@ -348,15 +350,13 @@ def test_delete_rows(self): cursor.execute(""" SELECT 'cd1', 'col' FROM StandardString1 WHERE KEY = 'kd' """) - colnames = [col_d[0] for col_d in cursor.description] - assert "cd1" in colnames - assert "col" in colnames + assert ['Row Key', 'cd1', 'col'] == [col_d[0] for col_d in cursor.description] cursor.execute("DELETE FROM StandardString1 WHERE KEY = 'kd'") cursor.execute(""" SELECT 'cd1', 'col' FROM StandardString1 WHERE KEY = 'kd' """) r = cursor.fetchone() - assert len(r) == 1 + assert ['kd', None, None] == r, r def test_create_keyspace(self): "create a new keyspace" diff --git a/test/system/test_thrift_server.py b/test/system/test_thrift_server.py index 9a398a7e7b61..9ac2ad10a0fb 100644 --- a/test/system/test_thrift_server.py +++ b/test/system/test_thrift_server.py @@ -1192,20 +1192,20 @@ def test_describe_snitch(self): def test_invalid_ks_names(self): def invalid_keyspace(): - client.system_add_keyspace(KsDef('in-valid', 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor':'1'}, [])) + client.system_add_keyspace(KsDef('in-valid', 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor':'1'}, cf_defs=[])) _expect_exception(invalid_keyspace, InvalidRequestException) def test_invalid_strategy_class(self): def add_invalid_keyspace(): - client.system_add_keyspace(KsDef('ValidKs', 'InvalidStrategyClass', {}, [])) + client.system_add_keyspace(KsDef('ValidKs', 'InvalidStrategyClass', {}, cf_defs=[])) exc = _expect_exception(add_invalid_keyspace, InvalidRequestException) s = str(exc) assert s.find("InvalidStrategyClass") > -1, s assert s.find("Unable to find replication strategy") > -1, s def update_invalid_keyspace(): - client.system_add_keyspace(KsDef('ValidKsForUpdate', 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor':'1'}, [])) - client.system_update_keyspace(KsDef('ValidKsForUpdate', 'InvalidStrategyClass', {}, [])) + client.system_add_keyspace(KsDef('ValidKsForUpdate', 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor':'1'}, cf_defs=[])) + client.system_update_keyspace(KsDef('ValidKsForUpdate', 'InvalidStrategyClass', {}, cf_defs=[])) exc = _expect_exception(update_invalid_keyspace, InvalidRequestException) s = str(exc) @@ -1222,7 +1222,7 @@ def invalid_cf(): def invalid_cf_inside_new_ks(): cf = CfDef('ValidKsName_invalid_cf', 'in-valid') _set_keyspace('system') - client.system_add_keyspace(KsDef('ValidKsName_invalid_cf', 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor': '1'}, [cf])) + client.system_add_keyspace(KsDef('ValidKsName_invalid_cf', 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor': '1'}, cf_defs=[cf])) _expect_exception(invalid_cf_inside_new_ks, InvalidRequestException) def test_system_cf_recreate(self): @@ -1234,7 +1234,7 @@ def test_system_cf_recreate(self): # create newcf = CfDef(keyspace, cf_name) - newks = KsDef(keyspace, 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor':'1'}, [newcf]) + newks = KsDef(keyspace, 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor':'1'}, cf_defs=[newcf]) client.system_add_keyspace(newks) _set_keyspace(keyspace) @@ -1261,7 +1261,7 @@ def test_system_keyspace_operations(self): keyspace = KsDef('CreateKeyspace', 'org.apache.cassandra.locator.SimpleStrategy', {'replication_factor': '10'}, - [CfDef('CreateKeyspace', 'CreateKsCf')]) + cf_defs=[CfDef('CreateKeyspace', 'CreateKsCf')]) client.system_add_keyspace(keyspace) newks = client.describe_keyspace('CreateKeyspace') assert 'CreateKsCf' in [x.name for x in newks.cf_defs] @@ -1272,7 +1272,7 @@ def test_system_keyspace_operations(self): modified_keyspace = KsDef('CreateKeyspace', 'org.apache.cassandra.locator.OldNetworkTopologyStrategy', {'replication_factor': '1'}, - []) + cf_defs=[]) client.system_update_keyspace(modified_keyspace) modks = client.describe_keyspace('CreateKeyspace') assert modks.strategy_class == modified_keyspace.strategy_class diff --git a/test/unit/org/apache/cassandra/cache/CacheProviderTest.java b/test/unit/org/apache/cassandra/cache/CacheProviderTest.java index 9f7e195032bb..334e6e3b486a 100644 --- a/test/unit/org/apache/cassandra/cache/CacheProviderTest.java +++ b/test/unit/org/apache/cassandra/cache/CacheProviderTest.java @@ -1,4 +1,25 @@ package org.apache.cassandra.cache; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.util.ArrayList; import java.util.List; diff --git a/test/unit/org/apache/cassandra/client/TestRingCache.java b/test/unit/org/apache/cassandra/client/TestRingCache.java index 8479207592d2..61ce6cf0ef37 100644 --- a/test/unit/org/apache/cassandra/client/TestRingCache.java +++ b/test/unit/org/apache/cassandra/client/TestRingCache.java @@ -103,7 +103,7 @@ public static void main(String[] args) throws Throwable // now, read the row back directly from the host owning the row locally tester.setup(firstEndpoint.getHostAddress(), DatabaseDescriptor.getRpcPort()); tester.thriftClient.set_keyspace(keyspace); - tester.thriftClient.insert(row, parent, new Column(ByteBufferUtil.bytes("col1"), ByteBufferUtil.bytes("val1"), 1), ConsistencyLevel.ONE); + tester.thriftClient.insert(row, parent, new Column(ByteBufferUtil.bytes("col1")).setValue(ByteBufferUtil.bytes("val1")).setTimestamp(1), ConsistencyLevel.ONE); Column column = tester.thriftClient.get(row, col, ConsistencyLevel.ONE).column; System.out.println("read row " + new String(row.array()) + " " + new String(column.name.array()) + ":" + new String(column.value.array()) + ":" + column.timestamp); } diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index c6b91f4cef26..49cfb76f650c 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -210,6 +210,35 @@ public void testIndexScan() throws IOException assert rows.isEmpty(); } + @Test + public void testLargeScan() throws IOException + { + RowMutation rm; + for (int i = 0; i < 100; i++) + { + rm = new RowMutation("Keyspace1", ByteBufferUtil.bytes("key" + i)); + rm.add(new QueryPath("Indexed1", null, ByteBufferUtil.bytes("birthdate")), ByteBufferUtil.bytes(34L), 0); + rm.add(new QueryPath("Indexed1", null, ByteBufferUtil.bytes("notbirthdate")), ByteBufferUtil.bytes((long) (i % 2)), 0); + rm.applyUnsafe(); + } + + IndexExpression expr = new IndexExpression(ByteBufferUtil.bytes("birthdate"), IndexOperator.EQ, ByteBufferUtil.bytes(34L)); + IndexExpression expr2 = new IndexExpression(ByteBufferUtil.bytes("notbirthdate"), IndexOperator.EQ, ByteBufferUtil.bytes(1L)); + IndexClause clause = new IndexClause(Arrays.asList(expr, expr2), ByteBufferUtil.EMPTY_BYTE_BUFFER, 100); + IFilter filter = new IdentityQueryFilter(); + IPartitioner p = StorageService.getPartitioner(); + Range range = new Range(p.getMinimumToken(), p.getMinimumToken()); + List rows = Table.open("Keyspace1").getColumnFamilyStore("Indexed1").scan(clause, range, filter); + + assert rows != null; + assert rows.size() == 50 : rows.size(); + Set keys = new HashSet(); + // extra check that there are no duplicate results -- see https://issues.apache.org/jira/browse/CASSANDRA-2406 + for (Row row : rows) + keys.add(row.key); + assert rows.size() == keys.size(); + } + @Test public void testIndexDeletions() throws IOException { diff --git a/test/unit/org/apache/cassandra/db/SystemTableTest.java b/test/unit/org/apache/cassandra/db/SystemTableTest.java index 9f097f1edc11..df44b784fb63 100644 --- a/test/unit/org/apache/cassandra/db/SystemTableTest.java +++ b/test/unit/org/apache/cassandra/db/SystemTableTest.java @@ -1,4 +1,25 @@ package org.apache.cassandra.db; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import java.net.InetAddress; import java.net.UnknownHostException; diff --git a/test/unit/org/apache/cassandra/db/marshal/UUIDTypeTest.java b/test/unit/org/apache/cassandra/db/marshal/UUIDTypeTest.java index cd01048d3f68..65b7fdd08324 100644 --- a/test/unit/org/apache/cassandra/db/marshal/UUIDTypeTest.java +++ b/test/unit/org/apache/cassandra/db/marshal/UUIDTypeTest.java @@ -1,4 +1,25 @@ package org.apache.cassandra.db.marshal; +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + import static org.junit.Assert.assertEquals; diff --git a/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java b/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java index 143efa70cb3b..44fb099d65c6 100644 --- a/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java +++ b/test/unit/org/apache/cassandra/service/EmbeddedCassandraServiceTest.java @@ -84,8 +84,10 @@ public void testEmbeddedCassandraService() cp.column = ByteBufferUtil.bytes("name"); // insert - client.insert(key_user_id, par, new Column(ByteBufferUtil.bytes("name"), - ByteBufferUtil.bytes("Ran"), timestamp), ConsistencyLevel.ONE); + client.insert(key_user_id, + par, + new Column(ByteBufferUtil.bytes("name")).setValue(ByteBufferUtil.bytes("Ran")).setTimestamp(timestamp), + ConsistencyLevel.ONE); // read ColumnOrSuperColumn got = client.get(key_user_id, cp, ConsistencyLevel.ONE); diff --git a/tools/stress/src/org/apache/cassandra/stress/Session.java b/tools/stress/src/org/apache/cassandra/stress/Session.java index 942c3467df7d..ac55955ad99e 100644 --- a/tools/stress/src/org/apache/cassandra/stress/Session.java +++ b/tools/stress/src/org/apache/cassandra/stress/Session.java @@ -73,6 +73,7 @@ public class Session availableOptions.addOption("R", "replication-strategy", true, "Replication strategy to use (only on insert if keyspace does not exist), default:org.apache.cassandra.locator.SimpleStrategy"); availableOptions.addOption("O", "strategy-properties", true, "Replication strategy properties in the following format :,:,..."); availableOptions.addOption("W", "no-replicate-on-write",false, "Set replicate_on_write to false for counters. Only counter add with CL=ONE will work"); + availableOptions.addOption("V", "average-size-values", false, "Generate column values of average rather than specific size"); } private int numKeys = 1000 * 1000; @@ -103,6 +104,7 @@ public class Session private String replicationStrategy = "org.apache.cassandra.locator.SimpleStrategy"; private Map replicationStrategyOptions = new HashMap(); + public final boolean averageSizeValues; // required by Gaussian distribution. protected int mean; @@ -260,6 +262,8 @@ else if (replicationStrategy.endsWith("SimpleStrategy")) if (cmd.hasOption("W")) replicateOnWrite = false; + + averageSizeValues = cmd.hasOption("V"); } catch (ParseException e) { diff --git a/tools/stress/src/org/apache/cassandra/stress/operations/IndexedRangeSlicer.java b/tools/stress/src/org/apache/cassandra/stress/operations/IndexedRangeSlicer.java index abb7f9cddfe1..24bdee8116e1 100644 --- a/tools/stress/src/org/apache/cassandra/stress/operations/IndexedRangeSlicer.java +++ b/tools/stress/src/org/apache/cassandra/stress/operations/IndexedRangeSlicer.java @@ -48,8 +48,8 @@ public void run(Cassandra.Client client) throws IOException int received = 0; - String startOffset = "0"; - ByteBuffer value = values.get(index % values.size()); + String startOffset = String.format(format, 0); + ByteBuffer value = values.get(1); // only C1 column is indexed IndexExpression expression = new IndexExpression(columnName, IndexOperator.EQ, value); diff --git a/tools/stress/src/org/apache/cassandra/stress/util/Operation.java b/tools/stress/src/org/apache/cassandra/stress/util/Operation.java index 6e5edb2aa213..5ec6f5b3ec07 100644 --- a/tools/stress/src/org/apache/cassandra/stress/util/Operation.java +++ b/tools/stress/src/org/apache/cassandra/stress/util/Operation.java @@ -55,11 +55,33 @@ public Operation(int idx) // Utility methods + protected List generateValues() + { + if (session.averageSizeValues) + { + return generateRandomizedValues(); + } + + List values = new ArrayList(); + + for (int i = 0; i < session.getCardinality(); i++) + { + String hash = getMD5(Integer.toString(i)); + int times = session.getColumnSize() / hash.length(); + int sumReminder = session.getColumnSize() % hash.length(); + + String value = new StringBuilder(multiplyString(hash, times)).append(hash.substring(0, sumReminder)).toString(); + values.add(ByteBuffer.wrap(value.getBytes())); + } + + return values; + } + /** * Generate values of average size specified by -S, up to cardinality specified by -C * @return Collection of the values */ - protected List generateValues() + protected List generateRandomizedValues() { List values = new ArrayList();