GG-25108 Control.sh commands like idle_verify require TASK_EXECUTE pe…

bin/control.bat

@@ -156,6 +156,11 @@ if %ERRORLEVEL% equ 0 (
     if "%JVM_OPTS%" == "" set JVM_OPTS=-Xms256m -Xmx1g
 )
 
+::
+:: Uncomment to enable experimental commands [--wal]
+::
+:: set JVM_OPTS=%JVM_OPTS% -DIGNITE_ENABLE_EXPERIMENTAL_COMMAND=true
+
 ::
 :: Uncomment the following GC settings if you see spikes in your throughput due to Garbage Collection.
 ::

examples/pom.xml

@@ -194,11 +194,6 @@
                     </exclusions>
                 </dependency>
 
-                <dependency>
-                    <groupId>org.apache.ignite</groupId>
-                    <artifactId>ignite-spark_2.10</artifactId>
-                    <version>${project.version}</version>
-                </dependency>
             </dependencies>
 
             <build>

modules/aop/src/test/java/org/apache/ignite/gridify/AbstractAopTest.java

@@ -25,6 +25,7 @@
 import org.apache.ignite.configuration.DeploymentMode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.deployment.local.LocalDeploymentSpi;
@@ -59,6 +60,8 @@ public abstract class AbstractAopTest extends GridCommonAbstractTest {
         cfg.setMetricsUpdateFrequency(500);
         cfg.setDeploymentMode(depMode);
 
+        cfg.setIncludeEventTypes(EventType.EVTS_ALL);
+
         return cfg;
     }
 

modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java

@@ -178,6 +178,7 @@ protected static class TestTree extends BPlusTree<Long, Long> {
             super(
                 "test",
                 cacheId,
+                null,
                 pageMem,
                 null,
                 new AtomicLong(),

modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jol/FileStoreHeapUtilizationJolBenchmark.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.benchmarks.jol;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.openjdk.jol.info.GraphLayout;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE;
+
+/**
+ *
+ */
+public class FileStoreHeapUtilizationJolBenchmark {
+    /** */
+    private void benchmark() throws IgniteCheckedException {
+        FilePageStoreFactory factory = new FileVersionCheckingFactory(
+            new AsyncFileIOFactory(),
+            new AsyncFileIOFactory(),
+            new DataStorageConfiguration()
+                .setPageSize(4096)
+        );
+
+        List<PageStore> stores = new LinkedList<>();
+
+        File workDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false);
+
+        for (int i = 0; i < 10000; i++) {
+            final int p = i;
+
+            PageStore ps = factory.createPageStore(
+                PageMemory.FLAG_DATA,
+                () -> getPartitionFilePath(workDir, p),
+                d -> { }
+            );
+
+            ps.ensure();
+
+            ps.write(0, ByteBuffer.allocate(256), 1, false);
+
+            stores.add(ps);
+        }
+
+        System.gc();
+
+        GraphLayout layout = GraphLayout.parseInstance(stores);
+
+        System.out.println("heap usage: " + layout.totalSize());
+
+        U.delete(workDir);
+    }
+
+    /** */
+    private Path getPartitionFilePath(File cacheWorkDir, int partId) {
+        return new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)).toPath();
+    }
+
+    /** */
+    public static void main(String[] args) throws Exception {
+        new FileStoreHeapUtilizationJolBenchmark().benchmark();
+    }
+}

modules/camel/pom.xml

@@ -35,7 +35,7 @@
     <url>http://ignite.apache.org</url>
 
     <properties>
-        <guava.version>18.0</guava.version>
+        <guava.version>25.1-jre</guava.version>
         <okhttp.version>2.5.0</okhttp.version>
     </properties>
 

modules/camel/src/test/java/org/apache/ignite/stream/camel/IgniteCamelStreamerTest.java

@@ -48,7 +48,9 @@
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.CacheEvent;
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.util.lang.GridMapEntry;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.lang.IgniteBiPredicate;
@@ -95,6 +97,11 @@ public IgniteCamelStreamerTest() {
         super(true);
     }
 
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        return super.getConfiguration(igniteInstanceName).setIncludeEventTypes(EventType.EVTS_ALL);
+    }
+
     @SuppressWarnings("unchecked")
     @Override public void beforeTest() throws Exception {
         grid().<Integer, String>getOrCreateCache(defaultCacheConfiguration());

modules/cassandra/store/pom.xml

@@ -39,7 +39,7 @@
         <cassandra-driver.version>3.0.0</cassandra-driver.version>
         <cassandra-all.version>3.3</cassandra-all.version>
         <netty.version>4.1.27.Final</netty.version>
-        <guava.version>19.0</guava.version>
+        <guava.version>25.1-jre</guava.version>
         <metrics-core.version>3.0.2</metrics-core.version>
     </properties>
 

modules/cassandra/store/src/main/java/org/apache/ignite/cache/store/cassandra/session/CassandraSessionImpl.java

@@ -648,17 +648,25 @@ private void createKeyspace(KeyValuePersistenceSettings settings) {
 
         while (attempt < CQL_EXECUTION_ATTEMPTS_COUNT) {
             try {
-                log.info("-----------------------------------------------------------------------");
-                log.info("Creating Cassandra keyspace '" + settings.getKeyspace() + "'");
-                log.info("-----------------------------------------------------------------------\n\n" +
-                    settings.getKeyspaceDDLStatement() + "\n");
-                log.info("-----------------------------------------------------------------------");
+                if (log.isInfoEnabled()) {
+                    log.info("-----------------------------------------------------------------------");
+                    log.info("Creating Cassandra keyspace '" + settings.getKeyspace() + "'");
+                    log.info("-----------------------------------------------------------------------\n\n" +
+                        settings.getKeyspaceDDLStatement() + "\n");
+                    log.info("-----------------------------------------------------------------------");
+                }
+
                 session().execute(settings.getKeyspaceDDLStatement());
-                log.info("Cassandra keyspace '" + settings.getKeyspace() + "' was successfully created");
+
+                if (log.isInfoEnabled())
+                    log.info("Cassandra keyspace '" + settings.getKeyspace() + "' was successfully created");
+
                 return;
             }
             catch (AlreadyExistsException ignored) {
-                log.info("Cassandra keyspace '" + settings.getKeyspace() + "' already exist");
+                if (log.isInfoEnabled())
+                    log.info("Cassandra keyspace '" + settings.getKeyspace() + "' already exist");
+
                 return;
             }
             catch (Throwable e) {
@@ -689,17 +697,25 @@ private void createTable(String table, KeyValuePersistenceSettings settings) {
 
         while (attempt < CQL_EXECUTION_ATTEMPTS_COUNT) {
             try {
-                log.info("-----------------------------------------------------------------------");
-                log.info("Creating Cassandra table '" + tableFullName + "'");
-                log.info("-----------------------------------------------------------------------\n\n" +
+                if (log.isInfoEnabled()) {
+                    log.info("-----------------------------------------------------------------------");
+                    log.info("Creating Cassandra table '" + tableFullName + "'");
+                    log.info("-----------------------------------------------------------------------\n\n" +
                         settings.getTableDDLStatement(table) + "\n");
-                log.info("-----------------------------------------------------------------------");
+                    log.info("-----------------------------------------------------------------------");
+                }
+
                 session().execute(settings.getTableDDLStatement(table));
-                log.info("Cassandra table '" + tableFullName + "' was successfully created");
+
+                if (log.isInfoEnabled())
+                    log.info("Cassandra table '" + tableFullName + "' was successfully created");
+
                 return;
             }
             catch (AlreadyExistsException ignored) {
-                log.info("Cassandra table '" + tableFullName + "' already exist");
+                if (log.isInfoEnabled())
+                    log.info("Cassandra table '" + tableFullName + "' already exist");
+
                 return;
             }
             catch (Throwable e) {
@@ -741,14 +757,19 @@ private void createTableIndexes(String table, KeyValuePersistenceSettings settin
 
         while (attempt < CQL_EXECUTION_ATTEMPTS_COUNT) {
             try {
-                log.info("-----------------------------------------------------------------------");
-                log.info("Creating indexes for Cassandra table '" + tableFullName + "'");
-                log.info("-----------------------------------------------------------------------");
+                if (log.isInfoEnabled()) {
+                    log.info("-----------------------------------------------------------------------");
+                    log.info("Creating indexes for Cassandra table '" + tableFullName + "'");
+                    log.info("-----------------------------------------------------------------------");
+                }
 
                 for (String statement : indexDDLStatements) {
                     try {
-                        log.info(statement);
-                        log.info("-----------------------------------------------------------------------");
+                        if (log.isInfoEnabled()) {
+                            log.info(statement);
+                            log.info("-----------------------------------------------------------------------");
+                        }
+
                         session().execute(statement);
                     }
                     catch (AlreadyExistsException ignored) {
@@ -759,7 +780,8 @@ private void createTableIndexes(String table, KeyValuePersistenceSettings settin
                     }
                 }
 
-                log.info("Indexes for Cassandra table '" + tableFullName + "' were successfully created");
+                if (log.isInfoEnabled())
+                    log.info("Indexes for Cassandra table '" + tableFullName + "' were successfully created");
 
                 return;
             }

modules/clients/src/test/java/org/apache/ignite/common/ClientSideCacheCreationDestructionWileTopologyChangeTest.java

@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.common;
+
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * {@inheritDoc} With topology events in parallel
+ */
+public class ClientSideCacheCreationDestructionWileTopologyChangeTest extends ClientSizeCacheCreationDestructionTest {
+    /** **/
+    private static final int MAX_NODES_CNT = 10;
+
+    /** **/
+    IgniteInternalFuture topChangeProcFut;
+
+    /** **/
+    AtomicBoolean procTopChanges = new AtomicBoolean(true);
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        topChangeProcFut = asyncTopologyChanges();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        procTopChanges.set(false);
+
+        topChangeProcFut.get();
+
+        super.afterTest();
+    }
+
+    /**
+     * @return {@code IgniteInternalFuture} to wait for topology process to stop in {@code afterTest()}.
+     */
+    private IgniteInternalFuture asyncTopologyChanges() {
+        return GridTestUtils.runAsync(() -> {
+            while (procTopChanges.get()) {
+                try {
+                    if (srv.cluster().nodes().size() < MAX_NODES_CNT)
+                        startGrid(UUID.randomUUID().toString());
+                }
+                catch (Exception e) {
+                    fail("Unable to add or remove node: " + e);
+                }
+            }
+        });
+    }
+}