Skip to content

Commit

Permalink
Fixed #1989 AS3 - Slow deobfuscation (AVM2DeobfuscatorSimpleOld)
Browse files Browse the repository at this point in the history
  • Loading branch information
jindrapetrik committed Mar 19, 2023
1 parent 8378042 commit dfe2d36
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 9 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ All notable changes to this project will be documented in this file.
- AS3 direct editation - interface method namespace
- AS3 p-code docs - deldescendants, negate_p operands
- AS3 p-code - IGNORE_REST method flag incorrectly shown as EXPLICIT
- [#1989] AS3 - Slow deobfuscation (AVM2DeobfuscatorSimpleOld)

### Changed
- AS1/2/3 P-code - format Number values with EcmaScript toString function
Expand Down Expand Up @@ -3001,6 +3002,7 @@ All notable changes to this project will be documented in this file.
[#1981]: https://www.free-decompiler.com/flash/issues/1981
[#1982]: https://www.free-decompiler.com/flash/issues/1982
[#1986]: https://www.free-decompiler.com/flash/issues/1986
[#1989]: https://www.free-decompiler.com/flash/issues/1989
[#1970]: https://www.free-decompiler.com/flash/issues/1970
[#1972]: https://www.free-decompiler.com/flash/issues/1972
[#1973]: https://www.free-decompiler.com/flash/issues/1973
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2798,10 +2798,16 @@ public void removeIgnored(MethodBody body) throws InterruptedException {
}

public int removeDeadCode(MethodBody body) throws InterruptedException {
return removeDeadCode(body, new Reference<>(-1));
}

public int removeDeadCode(MethodBody body, Reference<Integer> minChangedIpRef) throws InterruptedException {
HashMap<Integer, List<Integer>> refs = visitCode(body);
int cnt = 0;
Integer minChangedIp = -1;
for (int i = code.size() - 1; i >= 0; i--) {
if (refs.get(i).isEmpty()) {
minChangedIp = i;
code.get(i).setIgnored(true, 0);
cnt++;
}
Expand All @@ -2814,13 +2820,18 @@ public int removeDeadCode(MethodBody body) throws InterruptedException {
if (ins.definition instanceof JumpIns) {
if (ins.operands[0] == 0) {
ins.setIgnored(true, 0);
if (minChangedIp == -1 || minChangedIp > i) {
minChangedIp = i;
}
cnt++;
}
}
}

removeIgnored(body);

minChangedIpRef.setVal(minChangedIp);

return cnt;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
import com.jpexs.decompiler.graph.TranslateStack;
import com.jpexs.decompiler.graph.model.FalseItem;
import com.jpexs.decompiler.graph.model.TrueItem;
import com.jpexs.helpers.Reference;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -176,16 +177,18 @@ protected boolean removeObfuscationIfs(int classIndex, boolean isStatic, int scr
if (Thread.currentThread().isInterrupted()) {
throw new InterruptedException();
}

localData.scopeStack.clear();
localData.localRegs.clear();
localData.localRegAssignmentIps.clear();
localData.localRegs.clear();
initLocalRegs(localData, localReservedCount, body.max_regs);

if (executeInstructions(importantOffsets, staticRegs, body, abc, code, localData, i, code.code.size() - 1, null, inlineIns, jumpTargets)) {
//startover because dead code was removed and current ip is thus invalid
i = -1;
Reference<Integer> minChangedIpRef = new Reference<>(-1);
if (executeInstructions(importantOffsets, staticRegs, body, abc, code, localData, i, code.code.size() - 1, null, inlineIns, jumpTargets, minChangedIpRef)) {
int minChangedIp = minChangedIpRef.getVal();
if (minChangedIp < i + 1) {
i = minChangedIp - 1;
}
}
}

Expand Down Expand Up @@ -221,11 +224,13 @@ protected void initLocalRegs(AVM2LocalData localData, int localReservedCount, in
}
}

private boolean executeInstructions(Set<Long> importantOffsets, Map<Integer, GraphTargetItem> staticRegs, MethodBody body, ABC abc, AVM2Code code, AVM2LocalData localData, int idx, int endIdx, ExecutionResult result, List<AVM2Instruction> inlineIns, List<Integer> jumpTargets) throws InterruptedException {
private boolean executeInstructions(Set<Long> importantOffsets, Map<Integer, GraphTargetItem> staticRegs, MethodBody body, ABC abc, AVM2Code code, AVM2LocalData localData, int idx, int endIdx, ExecutionResult result, List<AVM2Instruction> inlineIns, List<Integer> jumpTargets, Reference<Integer> minChangedIpRef) throws InterruptedException {
List<GraphTargetItem> output = new ArrayList<>();

FixItemCounterTranslateStack stack = new FixItemCounterTranslateStack("");
int instructionsProcessed = 0;

int minChangedIp = Integer.MAX_VALUE;

endIdx = code.code.size() - 1;
while (true) {
Expand Down Expand Up @@ -270,9 +275,11 @@ private boolean executeInstructions(Set<Long> importantOffsets, Map<Integer, Gra
if (fins.definition instanceof NewFunctionIns) {
int fidx = code.code.indexOf(fins);
code.removeInstruction(fidx, body);
if (fidx < minChangedIp) minChangedIp = fidx;
}
int nidx = code.code.indexOf(ins);
code.removeInstruction(nidx, body);
if (nidx < minChangedIp) minChangedIp = nidx;
if (nins == null) {
idx = code.code.size();
} else {
Expand All @@ -295,6 +302,7 @@ private boolean executeInstructions(Set<Long> importantOffsets, Map<Integer, Gra
int regId = ((SetLocalTypeIns) def).getRegisterId(ins);
staticRegs.put(regId, localData.localRegs.get(regId).getNotCoerced());
code.replaceInstruction(idx, new AVM2Instruction(0, DeobfuscatePopIns.getInstance(), null), body);
if (idx < minChangedIp) minChangedIp = idx;

importantOffsets.clear();
importantOffsets.addAll(code.getImportantOffsets(body, false));
Expand All @@ -315,6 +323,7 @@ private boolean executeInstructions(Set<Long> importantOffsets, Map<Integer, Gra
}

code.replaceInstruction(idx, pushins, body);
if (idx < minChangedIp) minChangedIp = idx;
stack.push(staticRegs.get(regId));
ins = pushins;
def = ins.definition;
Expand Down Expand Up @@ -418,6 +427,7 @@ private boolean executeInstructions(Set<Long> importantOffsets, Map<Integer, Gra
boolean ifed = false;
if (def instanceof PopIns) {
code.replaceInstruction(idx, new AVM2Instruction(ins.getAddress(), DeobfuscatePopIns.getInstance(), null), body);
if (idx < minChangedIp) minChangedIp = idx;
idx++;
} else if (def instanceof JumpIns) {
long address = ins.getTargetAddress();
Expand Down Expand Up @@ -449,6 +459,7 @@ private boolean executeInstructions(Set<Long> importantOffsets, Map<Integer, Gra
AVM2Instruction jumpIns = new AVM2Instruction(0, AVM2Instructions.Jump, new int[]{0});
//jumpIns.operands[0] = ins.operands[0] /*- ins.getBytes().length*/ + jumpIns.getBytes().length;
code.replaceInstruction(idx, jumpIns, body);
if (idx < minChangedIp) minChangedIp = idx;
jumpIns.operands[0] = (int) (tarIns.getAddress() - jumpIns.getAddress() - jumpIns.getBytesLength());
for (int s = 0; s < stackCount; s++) {
code.insertInstruction(idx, new AVM2Instruction(ins.getAddress(), DeobfuscatePopIns.getInstance(), null), true, body);
Expand All @@ -457,6 +468,7 @@ private boolean executeInstructions(Set<Long> importantOffsets, Map<Integer, Gra
idx = code.adr2pos(jumpIns.getTargetAddress());
} else {
//System.err.println("replacing " + ins + " on " + idx + " with pop");
if (idx < minChangedIp) minChangedIp = idx;
code.replaceInstruction(idx, new AVM2Instruction(ins.getAddress(), DeobfuscatePopIns.getInstance(), null), body);
for (int s = 1 /*first is replaced*/; s < stackCount; s++) {
code.insertInstruction(idx, new AVM2Instruction(ins.getAddress(), DeobfuscatePopIns.getInstance(), null), true, body);
Expand All @@ -465,13 +477,24 @@ private boolean executeInstructions(Set<Long> importantOffsets, Map<Integer, Gra
idx++;
}

//this might be slow:-(, but makes importantOffsets relevant
code.removeDeadCode(body);
removeZeroJumps(code, body);
Reference<Integer> minChangedIp2Ref = new Reference<>(-1);
//this might be slow:-(, but makes importantOffsets relevant
code.removeDeadCode(body, minChangedIp2Ref);

if (minChangedIp2Ref.getVal() != -1 && minChangedIp2Ref.getVal() < minChangedIp) {
minChangedIp = minChangedIp2Ref.getVal();
}
minChangedIp2Ref.setVal(-1);
removeZeroJumps(code, body, minChangedIp2Ref);

if (minChangedIp2Ref.getVal() != -1 && minChangedIp2Ref.getVal() < minChangedIp) {
minChangedIp = minChangedIp2Ref.getVal();
}

importantOffsets.clear();
importantOffsets.addAll(code.getImportantOffsets(body, false));

minChangedIpRef.setVal(minChangedIp);
return true;
//endIdx = code.code.size()-1;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import com.jpexs.decompiler.flash.abc.types.MethodBody;
import com.jpexs.decompiler.flash.abc.types.traits.Trait;
import com.jpexs.decompiler.flash.helpers.SWFDecompilerAdapter;
import com.jpexs.helpers.Reference;
import java.util.Set;

/**
Expand All @@ -45,8 +46,13 @@
*/
public class AVM2DeobfuscatorZeroJumpsNullPushes extends SWFDecompilerAdapter {


protected boolean removeZeroJumps(AVM2Code code, MethodBody body) throws InterruptedException {
return removeZeroJumps(code, body, new Reference<>(-1));
}
protected boolean removeZeroJumps(AVM2Code code, MethodBody body, Reference<Integer> minChangedIpRef) throws InterruptedException {
boolean result = false;
int minChangedIp = -1;
for (int i = 0; i < code.code.size(); i++) {
AVM2Instruction ins = code.code.get(i);
if (ins.definition instanceof JumpIns) {
Expand All @@ -55,12 +61,16 @@ protected boolean removeZeroJumps(AVM2Code code, MethodBody body) throws Interru
throw new InterruptedException();
}

if (minChangedIp == -1) {
minChangedIp = i;
}
code.removeInstruction(i, body);
i--;
result = true;
}
}
}
minChangedIpRef.setVal(minChangedIp);
return result;
}

Expand Down

0 comments on commit dfe2d36

Please sign in to comment.