Add forward compatibility to TaggedFieldSerializer in issue #352.

README.md

@@ -559,7 +559,7 @@ BeanSerializer is very similar to FieldSerializer, except it uses bean getter an
 
 VersionFieldSerializer extends FieldSerializer and allows fields to have a `@Since(int)` annotation to indicate the version they were added. For a particular field, the value in `@Since` should never change once created. This is less flexible than FieldSerializer, which can handle most classes without needing annotations, but it provides backward compatibility. This means that new fields can be added, but removing, renaming or changing the type of any field will invalidate previous serialized bytes. VersionFieldSerializer has very little overhead (a single additional varint) compared to FieldSerializer.
 
-TaggedFieldSerializer extends FieldSerializer to only serialize fields that have a `@Tag(int)` annotation, providing backward compatibility so new fields can be added. TaggedFieldSerializer has two advantages over VersionFieldSerializer: 1) fields can be renamed and 2) fields marked with the `@Deprecated` annotation will be ignored when reading old bytes and won't be written to new bytes. Deprecation effectively removes the field from serialization, though the field and `@Tag` annotation must remain in the class. Deprecated fields can optionally be made private and/or renamed so they don't clutter the class (eg, `ignored`, `ignored2`). For these reasons, TaggedFieldSerializer generally provides more flexibility for classes to evolve. The downside is that it has a small amount of additional overhead compared to VersionFieldSerializer (an additional varint per field).
+TaggedFieldSerializer extends FieldSerializer to only serialize fields that have a `@Tag(int)` annotation, providing backward compatibility so new fields can be added. And it also provides forward compatibility by `setIgnoreUnknownTags(true)`, thus any unknown field tags will be ignored. TaggedFieldSerializer has two advantages over VersionFieldSerializer: 1) fields can be renamed and 2) fields marked with the `@Deprecated` annotation will be ignored when reading old bytes and won't be written to new bytes. Deprecation effectively removes the field from serialization, though the field and `@Tag` annotation must remain in the class. Deprecated fields can optionally be made private and/or renamed so they don't clutter the class (eg, `ignored`, `ignored2`). For these reasons, TaggedFieldSerializer generally provides more flexibility for classes to evolve. The downside is that it has a small amount of additional overhead compared to VersionFieldSerializer (an additional varint per field).
 
 CompatibleFieldSerializer extends FieldSerializer to provide both forward and backward compatibility, meaning fields can be added or removed without invalidating previously serialized bytes. Changing the type of a field is not supported. Like FieldSerializer, it can serialize most classes without needing annotations. The forward and backward compatibility comes at a cost: the first time the class is encountered in the serialized bytes, a simple schema is written containing the field name strings. Also, during serialization and deserialization buffers are allocated to perform chunked encoding. This is what enables CompatibleFieldSerializer to skip bytes for fields it does not know about. When Kryo is configured to use references, there can be a [problem](https://github.com/EsotericSoftware/kryo/issues/286#issuecomment-74870545) with CompatibleFieldSerializer if a field is removed.
 

src/com/esotericsoftware/kryo/Kryo.java

@@ -49,6 +49,7 @@
 import com.esotericsoftware.kryo.serializers.FieldSerializerConfig;
 import com.esotericsoftware.kryo.serializers.OptionalSerializers;
 import com.esotericsoftware.kryo.serializers.GenericsResolver;
+import com.esotericsoftware.kryo.serializers.TaggedFieldSerializerConfig;
 import com.esotericsoftware.kryo.serializers.TimeSerializers;
 import org.objenesis.instantiator.ObjectInstantiator;
 import org.objenesis.strategy.InstantiatorStrategy;
@@ -150,6 +151,7 @@ public class Kryo {
 	private GenericsResolver genericsResolver = new GenericsResolver();
 
 	private FieldSerializerConfig fieldSerializerConfig = new FieldSerializerConfig();
+	private TaggedFieldSerializerConfig taggedFieldSerializerConfig = new TaggedFieldSerializerConfig();
 
 	private StreamFactory streamFactory;
 
@@ -1066,6 +1068,10 @@ public FieldSerializerConfig getFieldSerializerConfig() {
 		return fieldSerializerConfig;
 	}
 
+	public TaggedFieldSerializerConfig getTaggedFieldSerializerConfig() {
+		return taggedFieldSerializerConfig;
+	}
+
 	/** Sets the reference resolver and enables references. */
 	public void setReferenceResolver (ReferenceResolver referenceResolver) {
 		if (referenceResolver == null) throw new IllegalArgumentException("referenceResolver cannot be null.");

src/com/esotericsoftware/kryo/serializers/FieldSerializer.java

@@ -70,7 +70,7 @@ public class FieldSerializer<T> extends Serializer<T> implements Comparator<Fiel
 	/** type variables declared for this type */
 	final TypeVariable[] typeParameters;
 	final Class componentType;
-	private final FieldSerializerConfig config;
+	protected final FieldSerializerConfig config;
 	private CachedField[] fields = new CachedField[0];
 	private CachedField[] transientFields = new CachedField[0];
 	protected HashSet<CachedField> removedFields = new HashSet();
@@ -133,6 +133,10 @@ public FieldSerializer (Kryo kryo, Class type) {
 	}
 
 	public FieldSerializer (Kryo kryo, Class type, Class[] generics) {
+		this(kryo, type, generics, kryo.getFieldSerializerConfig().clone());
+	}
+
+	protected FieldSerializer (Kryo kryo, Class type, Class[] generics, FieldSerializerConfig config) {
 		this.kryo = kryo;
 		this.type = type;
 		this.generics = generics;
@@ -141,7 +145,7 @@ public FieldSerializer (Kryo kryo, Class type, Class[] generics) {
 			this.componentType = type.getComponentType();
 		else
 			this.componentType = null;
-		this.config = kryo.getFieldSerializerConfig().clone();
+		this.config = config;
 		this.genericsUtil = new FieldSerializerGenericsUtil(this);
 		this.unsafeUtil = FieldSerializerUnsafeUtil.Factory.getInstance(this);
 		this.annotationsUtil = new FieldSerializerAnnotationsUtil(this);

src/com/esotericsoftware/kryo/serializers/FieldSerializerConfig.java

@@ -27,7 +27,7 @@
 /** Configuration for FieldSerializer instances. To configure defaults for new FieldSerializer instances
  * use {@link Kryo#getFieldSerializerConfig()}, to configure a specific FieldSerializer instance use setters
  * for configuration settings on this specific FieldSerializer. */
-public final class FieldSerializerConfig implements Cloneable {
+public class FieldSerializerConfig implements Cloneable {
     private boolean fieldsCanBeNull = true, setFieldsAsAccessible = true;
     private boolean ignoreSyntheticFields = true;
     private boolean fixedFieldTypes;

src/com/esotericsoftware/kryo/serializers/TaggedFieldSerializer.java

@@ -19,13 +19,16 @@
 
 package com.esotericsoftware.kryo.serializers;
 
-import static com.esotericsoftware.minlog.Log.*;
+import static com.esotericsoftware.minlog.Log.TRACE;
+import static com.esotericsoftware.minlog.Log.trace;
 
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Comparator;
 
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.KryoException;
@@ -48,7 +51,24 @@ public class TaggedFieldSerializer<T> extends FieldSerializer<T> {
 	private boolean[] deprecated;
 
 	public TaggedFieldSerializer (Kryo kryo, Class type) {
-		super(kryo, type);
+		super(kryo, type, null, kryo.getTaggedFieldSerializerConfig().clone());
+	}
+
+	/** Tells Kryo, if should ignore unknown field tags when using TaggedFieldSerializer. Already existing serializer instances
+	 * are not affected by this setting.
+	 *
+	 * <p>
+	 * By default, Kryo will throw KryoException if encounters unknown field tags.
+	 * </p>
+	 *
+	 * @param ignoreUnknownTags if true, unknown field tags will be ignored. Otherwise KryoException will be thrown */
+	public void setIgnoreUnknownTags (boolean ignoreUnknownTags) {
+		((TaggedFieldSerializerConfig) config).setIgnoreUnknownTags(ignoreUnknownTags);
+		rebuildCachedFields();
+	}
+
+	public boolean isIgnoreUnkownTags() {
+		return ((TaggedFieldSerializerConfig) config).isIgnoreUnknownTags();
 	}
 
 	protected void initializeCachedFields () {
@@ -66,6 +86,13 @@ protected void initializeCachedFields () {
 		tags = new int[fields.length];
 		deprecated = new boolean[fields.length];
 		writeFieldCount = fields.length;
+
+		// fields are sorted to ensure write order: tag 0, tag 1, ... , tag N
+		Arrays.sort(fields, new Comparator<CachedField>() {
+			public int compare(CachedField o1, CachedField o2) {
+				return o1.getField().getAnnotation(Tag.class).value() - o2.getField().getAnnotation(Tag.class).value();
+			}
+		});
 		for (int i = 0, n = fields.length; i < n; i++) {
 			Field field = fields[i].getField();
 			tags[i] = field.getAnnotation(Tag.class).value();
@@ -114,8 +141,11 @@ public T read (Kryo kryo, Input input, Class<T> type) {
 					break;
 				}
 			}
-			if (cachedField == null) throw new KryoException("Unknown field tag: " + tag + " (" + getType().getName() + ")");
-			cachedField.read(input, object);
+			if (cachedField == null) {
+				if (!isIgnoreUnkownTags()) throw new KryoException("Unknown field tag: " + tag + " (" + getType().getName() + ")");
+			} else {
+				cachedField.read(input, object);
+			}
 		}
 		return object;
 	}

src/com/esotericsoftware/kryo/serializers/TaggedFieldSerializerConfig.java

@@ -0,0 +1,52 @@
+/* Copyright (c) 2008, Nathan Sweet
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided with the distribution.
+ * - Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
+ * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
+
+package com.esotericsoftware.kryo.serializers;
+
+import static com.esotericsoftware.minlog.Log.TRACE;
+import static com.esotericsoftware.minlog.Log.trace;
+
+/** Configuration for TaggedFieldSerializer instances. */
+public class TaggedFieldSerializerConfig extends FieldSerializerConfig {
+
+    /** ignore unknown field tags when using TaggedFieldSerializer. */
+    private boolean ignoreUnknownTags = false;
+
+    /** Tells Kryo, if should ignore unknown field tags when using TaggedFieldSerializer. Already existing serializer instances
+     * are not affected by this setting.
+     *
+     * <p>
+     * By default, Kryo will throw KryoException if encounters unknown field tags.
+     * </p>
+     *
+     * @param ignoreUnknownTags if true, unknown field tags will be ignored. Otherwise KryoException will be thrown */
+    public void setIgnoreUnknownTags (boolean ignoreUnknownTags) {
+        this.ignoreUnknownTags = ignoreUnknownTags;
+        if (TRACE) trace("kryo.TaggedFieldSerializerConfig", "setIgnoreUnknownTags: " + ignoreUnknownTags);
+    }
+
+    public boolean isIgnoreUnknownTags () {
+        return ignoreUnknownTags;
+    }
+
+    @Override
+    protected TaggedFieldSerializerConfig clone() {
+        return (TaggedFieldSerializerConfig) super.clone();
+    }
+}

test/com/esotericsoftware/kryo/TaggedFieldSerializerTest.java

@@ -15,12 +15,14 @@
  * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
-
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */
+
 package com.esotericsoftware.kryo;
 
 import java.io.FileNotFoundException;
 
+import org.objenesis.strategy.StdInstantiatorStrategy;
+
 import com.esotericsoftware.kryo.serializers.TaggedFieldSerializer;
 import com.esotericsoftware.kryo.serializers.TaggedFieldSerializer.Tag;
 
@@ -61,6 +63,54 @@ public void testAddedField () throws FileNotFoundException {
 		assertEquals(object1, object2);
 	}
 
+	/**
+	 * Serializes an object with two fields "b" and "a", then tries to read the generated byte stream into an object whose class has only a field "b". Test
+	 * fails because kryo writes fields sorted alphabetically.
+	 */
+	public void testSerializeWithNewDisorderedField() {
+		kryo.setReferences(true);
+		kryo.getTaggedFieldSerializerConfig().setIgnoreUnknownTags(true);
+		kryo.setDefaultSerializer(TaggedFieldSerializer.class);
+		kryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(new StdInstantiatorStrategy()));
+		kryo.register(RootWithNewDisorderedField.class, 50);
+
+		Kryo newKryo = new Kryo();
+		newKryo.getTaggedFieldSerializerConfig().setIgnoreUnknownTags(true);
+		newKryo.setDefaultSerializer(TaggedFieldSerializer.class);
+		newKryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(new StdInstantiatorStrategy()));
+		newKryo.register(Root.class, 50);
+
+		RootWithNewDisorderedField rootWithNewDisorderedField = new RootWithNewDisorderedField(33, "xxxxx");
+		roundTrip(13, 16, rootWithNewDisorderedField);
+
+		Root root = (Root) newKryo.readClassAndObject(input);
+		assertEquals(rootWithNewDisorderedField.b, root.b);
+	}
+
+	/**
+	 * Serializes a Class with two fields "b" and "c", then tries to read the byte stream into a class with only field "b". Test succeeds because kryo writes
+	 * fields sorted alphabetically.
+	 */
+	public void testSerializeWithNewOrderedField() {
+		kryo.setReferences(true);
+		kryo.getTaggedFieldSerializerConfig().setIgnoreUnknownTags(true);
+		kryo.setDefaultSerializer(TaggedFieldSerializer.class);
+		kryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(new StdInstantiatorStrategy()));
+		kryo.register(RootWithNewOrderedField.class, 50);
+
+		Kryo newKryo = new Kryo();
+		newKryo.getTaggedFieldSerializerConfig().setIgnoreUnknownTags(true);
+		newKryo.setDefaultSerializer(TaggedFieldSerializer.class);
+		newKryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(new StdInstantiatorStrategy()));
+		newKryo.register(Root.class, 50);
+
+		RootWithNewOrderedField rootWithNewOrderedField = new RootWithNewOrderedField(33, "xxxxx");
+		roundTrip(13, 16, rootWithNewOrderedField);
+
+		Root root = (Root) newKryo.readClassAndObject(input);
+		assertEquals(rootWithNewOrderedField.b, root.b);
+	}
+
 	static public class TestClass {
 		@Tag(0) public String text = "something";
 		@Tag(1) public int moo = 120;
@@ -91,4 +141,81 @@ public boolean equals (Object obj) {
 	static public class AnotherClass {
 		@Tag(1) String value;
 	}
-}
+
+	private static class Root {
+		@TaggedFieldSerializer.Tag(0)
+		private Integer b;
+	}
+
+	private static class RootWithNewDisorderedField {
+
+		@TaggedFieldSerializer.Tag(0)
+		private Integer b;
+
+		/**
+		 *  Because it starts with an 'a' kryo writes it before b field
+		 */
+		@TaggedFieldSerializer.Tag(1)
+		private String a;
+
+		public RootWithNewDisorderedField(Integer b, String a) {
+			this.b = b;
+			this.a = a;
+		}
+
+		@Override
+		public boolean equals(Object o) {
+			if (this == o) return true;
+			if (!(o instanceof RootWithNewDisorderedField)) return false;
+
+			RootWithNewDisorderedField that = (RootWithNewDisorderedField) o;
+
+			if (b != null ? !b.equals(that.b) : that.b != null) return false;
+			return a != null ? a.equals(that.a) : that.a == null;
+
+		}
+
+		@Override
+		public int hashCode() {
+			int result = b != null ? b.hashCode() : 0;
+			result = 31 * result + (a != null ? a.hashCode() : 0);
+			return result;
+		}
+	}
+
+	private static class RootWithNewOrderedField {
+
+		@TaggedFieldSerializer.Tag(0)
+		private Integer b;
+
+		/**
+		 *  Because it starts with a 'c' kryo writes it after b field
+		 */
+		@TaggedFieldSerializer.Tag(1)
+		private String c;
+
+		public RootWithNewOrderedField(Integer b, String c) {
+			this.b = b;
+			this.c = c;
+		}
+
+		@Override
+		public boolean equals(Object o) {
+			if (this == o) return true;
+			if (!(o instanceof RootWithNewOrderedField)) return false;
+
+			RootWithNewOrderedField that = (RootWithNewOrderedField) o;
+
+			if (b != null ? !b.equals(that.b) : that.b != null) return false;
+			return c != null ? c.equals(that.c) : that.c == null;
+
+		}
+
+		@Override
+		public int hashCode() {
+			int result = b != null ? b.hashCode() : 0;
+			result = 31 * result + (c != null ? c.hashCode() : 0);
+			return result;
+		}
+	}
+}