Fix buffer overflow in TransformationListListNC

To save memory, when calculating the size of the transformation, indices where src[i]=ran[i] are skipped. Therefore these must also be skipped when filling in the transformation later.
gap-system · May 18, 2019 · f91b816 · f91b816
1 parent 7162a59
commit f91b816
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 4 deletions.
diff --git a/src/trans.cc b/src/trans.cc
@@ -476,8 +476,11 @@ static Obj FuncTransformationListListNC(Obj self, Obj src, Obj ran)
             ptf2[i] = i;
         }
         for (i = LEN_LIST(src); 1 <= i; i--) {
-            ptf2[INT_INTOBJ(ELM_LIST(src, i)) - 1] =
-                INT_INTOBJ(ELM_LIST(ran, i)) - 1;
+            s = INT_INTOBJ(ELM_LIST(src, i));
+            r = INT_INTOBJ(ELM_LIST(ran, i));
+            if(s != r) {
+                ptf2[s - 1] = r - 1;
+            }
         }
     }
     else {
@@ -487,8 +490,11 @@ static Obj FuncTransformationListListNC(Obj self, Obj src, Obj ran)
             ptf4[i] = i;
         }
         for (i = LEN_LIST(src); 1 <= i; i--) {
-            ptf4[INT_INTOBJ(ELM_LIST(src, i)) - 1] =
-                INT_INTOBJ(ELM_LIST(ran, i)) - 1;
+            s = INT_INTOBJ(ELM_LIST(src, i));
+            r = INT_INTOBJ(ELM_LIST(ran, i));
+            if(s != r) {
+                ptf4[s - 1] = r - 1;
+            }
         }
     }
     return f;

diff --git a/tst/testinstall/trans.tst b/tst/testinstall/trans.tst
@@ -57,6 +57,12 @@ Error, TransformationListListNC: <ran> must be a small list (not a permutation\
  (small))
 gap> TransformationListListNC([], []);
 IdentityTransformation
+gap> TransformationListListNC([1..100000], [1..100000]);
+IdentityTransformation
+gap> TransformationListList([1..1000000], Concatenation([100000], [2..1000000]));
+<transformation on 100000 pts with rank 99999>
+gap> TransformationListList([1..1000000], Concatenation([2], [2..1000000]));
+Transformation( [ 2, 2 ] )
 
 # Test DegreeOfTransformation
 gap> f := TransformationListListNC([1, 2], [1, 1]) ^ (3, 4);;