IEP-4 Baseline topology for persistent caches (Phase 1)

Contributed by:
    Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
    Dmitry Pavlov <dpavlov.spb@gmail.com>
    Eduard Shangareev <eduard.shangareev@gmail.com>
    Ilya Lantukh <ilantukh@gridgain.com>
    Ivan Rakov <ivan.glukos@gmail.com>
    Sergey Chugunov <sergey.chugunov@gmail.com>

RELEASE_NOTES.txt

@@ -1,6 +1,14 @@
 Apache Ignite Release Notes
 ===========================
 
+Apache Ignite In-Memory Data Fabric 2.4
+---------------------------------------
+Ignite:
+* Introduced Baseline Affinity Topology
+* Added ability to convert WAL to human-readable form
+* Added support for uninterruptible writes for WAL and storage
+* Local scan performance was improved up to 2 times in certain use-cases
+
 Apache Ignite In-Memory Data Fabric 2.3
 ---------------------------------------
 Ignite:

modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java

@@ -46,6 +46,9 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
     /** */
     protected static final String TEST_CACHE_NAME = DummyPersistenceCompatibilityTest.class.getSimpleName();
 
+    /** */
+    protected volatile boolean compactFooter;
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
@@ -59,11 +62,17 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
 
         cfg.setPeerClassLoadingEnabled(false);
 
-        DataStorageConfiguration memCfg = new DataStorageConfiguration()
-            .setDefaultDataRegionConfiguration(
-                new DataRegionConfiguration().setPersistenceEnabled(true));
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                ));
 
-        cfg.setDataStorageConfiguration(memCfg);
+        cfg.setBinaryConfiguration(
+            new BinaryConfiguration()
+                .setCompactFooter(compactFooter)
+        );
 
         return cfg;
     }
@@ -102,7 +111,11 @@ public void testNodeStartByOldVersionPersistenceData_2_3() throws Exception {
      * @throws Exception If failed.
      */
     protected void doTestStartupWithOldVersion(String igniteVer, boolean compactFooter) throws Exception {
+        boolean prev = this.compactFooter;
+
         try {
+            this.compactFooter = compactFooter;
+
             startGrid(1, igniteVer, new ConfigurationClosure(compactFooter), new PostStartupClosure());
 
             stopAllGrids();
@@ -113,10 +126,12 @@ protected void doTestStartupWithOldVersion(String igniteVer, boolean compactFoot
 
             ignite.active(true);
 
-            validateResultingCacheData(ignite.getOrCreateCache(TEST_CACHE_NAME));
+            validateResultingCacheData(ignite.cache(TEST_CACHE_NAME));
         }
         finally {
             stopAllGrids();
+
+            this.compactFooter = prev;
         }
     }
 

modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/util/MavenUtils.java

@@ -39,6 +39,12 @@ public class MavenUtils {
     /** Path to Maven local repository. For caching. */
     private static String locRepPath = null;
 
+    /** */
+    private static final String GG_MVN_REPO = "http://www.gridgainsystems.com/nexus/content/repositories/external";
+
+    /** Set this flag to true if running PDS compatibility tests locally. */
+    private static boolean useGgRepo;
+
     /**
      * Gets a path to an artifact with given version and groupId=org.apache.ignite and artifactId={@code artifactName}.
      * <br>
@@ -123,7 +129,7 @@ public static String getMavenLocalRepositoryPath() throws Exception {
      * @throws Exception In case of an error.
      */
     private static String defineMavenLocalRepositoryPath() throws Exception {
-        String output = exec("mvn help:effective-settings");
+        String output = exec(buildMvnCommand() + " help:effective-settings");
 
         int endTagPos = output.indexOf("</localRepository>");
 
@@ -141,7 +147,8 @@ private static String defineMavenLocalRepositoryPath() throws Exception {
     private static void downloadArtifact(String artifact) throws Exception {
         X.println("Downloading artifact... Identifier: " + artifact);
 
-        exec("mvn dependency:get -Dartifact=" + artifact);
+        exec(buildMvnCommand() + " dependency:get -Dartifact=" + artifact +
+            (useGgRepo ? " -DremoteRepositories=" + GG_MVN_REPO : ""));
 
         X.println("Download is finished");
     }
@@ -193,4 +200,19 @@ private static String exec(String cmd) throws Exception {
             throw e;
         }
     }
+
+    /**
+     * @return Maven executable command.
+     */
+    private static String buildMvnCommand() {
+        String m2Home = System.getenv("M2_HOME");
+
+        if (m2Home == null)
+            m2Home = System.getProperty("M2_HOME");
+
+        if (m2Home == null)
+            return "mvn";
+
+        return m2Home + "/bin/mvn" ;
+    }
 }

modules/core/src/main/java/org/apache/ignite/Ignite.java

@@ -659,15 +659,19 @@ public <T> IgniteQueue<T> queue(String name, int cap, @Nullable CollectionConfig
      * Checks Ignite grid is active or not active.
      *
      * @return {@code True} if grid is active. {@code False} If grid is not active.
+     * @deprecated Use {@link IgniteCluster#active()} instead.
      */
+    @Deprecated
     public boolean active();
 
     /**
      * Changes Ignite grid state to active or inactive.
      *
      * @param active If {@code True} start activation process. If {@code False} start deactivation process.
      * @throws IgniteException If there is an already started transaction or lock in the same thread.
+     * @deprecated Use {@link IgniteCluster#active(boolean)} instead.
      */
+    @Deprecated
     public void active(boolean active);
 
     /**

modules/core/src/main/java/org/apache/ignite/IgniteCluster.java

@@ -22,6 +22,7 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.cluster.ClusterStartNodeResult;
@@ -427,6 +428,44 @@ public IgniteFuture<Collection<ClusterStartNodeResult>> startNodesAsync(Collecti
      */
     @Nullable public IgniteFuture<?> clientReconnectFuture();
 
+    /**
+     * Checks Ignite grid is active or not active.
+     *
+     * @return {@code True} if grid is active. {@code False} If grid is not active.
+     */
+    public boolean active();
+
+    /**
+     * Changes Ignite grid state to active or inactive.
+     *
+     * @param active If {@code True} start activation process. If {@code False} start deactivation process.
+     * @throws IgniteException If there is an already started transaction or lock in the same thread.
+     */
+    public void active(boolean active);
+
+    /**
+     * Gets current baseline topology. If baseline topology was not set, will return {@code null}.
+     *
+     * @return Collection of nodes included to the current baseline topology.
+     */
+    @Nullable public Collection<BaselineNode> currentBaselineTopology();
+
+    /**
+     * Sets baseline topology. The cluster must be activated for this method to be called.
+     *
+     * @param baselineTop A collection of nodes to be included to the baseline topology.
+     */
+    public void setBaselineTopology(Collection<? extends BaselineNode> baselineTop);
+
+    /**
+     * Sets baseline topology constructed from the cluster topology of the given version (the method succeeds
+     * only if the cluster topology has not changed). All client and daemon nodes will be filtered out of the
+     * resulting baseline.
+     *
+     * @param topVer Topology version to set.
+     */
+    public void setBaselineTopology(long topVer);
+
     /** {@inheritDoc} */
     @Deprecated
     @Override public IgniteCluster withAsync();

modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java

@@ -759,6 +759,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_WAL_SERIALIZER_VERSION = "IGNITE_WAL_SERIALIZER_VERSION";
 
+    /**
+     * If the property is set Ignite will use legacy node comparator (based on node order) inste
+     *
+     * Default value is {@code false}.
+     */
+    public static final String IGNITE_USE_LEGACY_NODE_COMPARATOR = "IGNITE_USE_LEGACY_NODE_COMPARATOR";
+
     /**
      * Property that indicates should be mapped byte buffer used or not.
      * Possible values: {@code true} and {@code false}.
@@ -772,6 +779,16 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID = "IGNITE_DATA_STORAGE_FOLDER_BY_CONSISTENT_ID";
 
+    /**
+     * Default value is {@code false}.
+     */
+    public static final String IGNITE_WAL_DEBUG_LOG_ON_RECOVERY = "IGNITE_WAL_DEBUG_LOG_ON_RECOVERY";
+
+    /**
+     * Number of checkpoint history entries held in memory.
+     */
+    public static final String IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE = "IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE";
+
     /**
      * If this property is set to {@code true} enable logging in {@link GridClient}.
      */

modules/core/src/main/java/org/apache/ignite/cluster/BaselineNode.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cluster;
+
+import java.util.Map;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Interface representing a single node from baseline topology.
+ */
+public interface BaselineNode {
+    /**
+     * Gets consistent globally unique node ID. This method returns consistent node ID which
+     * survives node restarts.
+     *
+     * @return Consistent globally unique node ID.
+     */
+    public Object consistentId();
+
+    /**
+     * Gets a node attribute. Attributes are assigned to nodes at startup
+     * via {@link IgniteConfiguration#getUserAttributes()} method.
+     * <p>
+     * The system adds the following attributes automatically:
+     * <ul>
+     * <li>{@code {@link System#getProperties()}} - All system properties.</li>
+     * <li>{@code {@link System#getenv(String)}} - All environment properties.</li>
+     * <li>All attributes defined in {@link org.apache.ignite.internal.IgniteNodeAttributes}</li>
+     * </ul>
+     * <p>
+     * Note that attributes cannot be changed at runtime.
+     *
+     * @param <T> Attribute Type.
+     * @param name Attribute name. <b>Note</b> that attribute names starting with
+     *      {@code org.apache.ignite} are reserved for internal use.
+     * @return Attribute value or {@code null}.
+     */
+    @Nullable
+    public <T> T attribute(String name);
+
+    /**
+     * Gets all node attributes. Attributes are assigned to nodes at startup
+     * via {@link IgniteConfiguration#getUserAttributes()} method.
+     * <p>
+     * The system adds the following attributes automatically:
+     * <ul>
+     * <li>{@code {@link System#getProperties()}} - All system properties.</li>
+     * <li>{@code {@link System#getenv(String)}} - All environment properties.</li>
+     * <li>All attributes defined in {@link org.apache.ignite.internal.IgniteNodeAttributes}</li>
+     * </ul>
+     * <p>
+     * Note that attributes cannot be changed at runtime.
+     *
+     * @return All node attributes.
+     */
+    public Map<String, Object> attributes();
+}

modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java

@@ -101,7 +101,7 @@
  * that comes with JDK as it also provides ability to view any node parameter
  * as a graph.
  */
-public interface ClusterNode {
+public interface ClusterNode extends BaselineNode {
     /**
      * Gets globally unique node ID. A new ID is generated every time a node restarts.
      *
@@ -115,7 +115,7 @@ public interface ClusterNode {
      *
      * @return Consistent globally unique node ID.
      */
-    public Object consistentId();
+    @Override public Object consistentId();
 
     /**
      * Gets a node attribute. Attributes are assigned to nodes at startup
@@ -135,7 +135,7 @@ public interface ClusterNode {
      *      {@code org.apache.ignite} are reserved for internal use.
      * @return Attribute value or {@code null}.
      */
-    @Nullable public <T> T attribute(String name);
+    @Override @Nullable public <T> T attribute(String name);
 
     /**
      * Gets metrics snapshot for this node. Note that node metrics are constantly updated
@@ -167,7 +167,7 @@ public interface ClusterNode {
      *
      * @return All node attributes.
      */
-    public Map<String, Object> attributes();
+    @Override public Map<String, Object> attributes();
 
     /**
      * Gets collection of addresses this node is known by.

modules/core/src/main/java/org/apache/ignite/configuration/AtomicConfiguration.java

@@ -29,7 +29,7 @@
  */
 public class AtomicConfiguration {
     /** */
-    public static final int DFLT_BACKUPS = 0;
+    public static final int DFLT_BACKUPS = 1;
 
     /** */
     public static final CacheMode DFLT_CACHE_MODE = PARTITIONED;

modules/core/src/main/java/org/apache/ignite/configuration/DataRegionConfiguration.java

@@ -19,6 +19,7 @@
 import java.io.Serializable;
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.mxbean.DataRegionMetricsMXBean;
 
 import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
@@ -431,4 +432,9 @@ public DataRegionConfiguration setCheckpointPageBufferSize(long checkpointPageBu
 
         return this;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DataRegionConfiguration.class, this);
+    }
 }

modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java

@@ -23,6 +23,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
@@ -209,7 +210,7 @@ public class DataStorageConfiguration implements Serializable {
     private int walTlbSize = DFLT_TLB_SIZE;
 
     /** WAl buffer size. */
-    private int walBuffSize/* = DFLT_WAL_BUFF_SIZE*/;
+    private int walBuffSize;
 
     /** Wal flush frequency in milliseconds. */
     private long walFlushFreq = DFLT_WAL_FLUSH_FREQ;
@@ -902,4 +903,9 @@ public DataStorageConfiguration setWalCompactionEnabled(boolean walCompactionEna
 
         return this;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DataStorageConfiguration.class, this);
+    }
 }