Integration of features extraction, generation and algorithms evaluation

ArchitectureReloaded.iml

@@ -5,6 +5,7 @@
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/.gradle" />
       <excludeFolder url="file://$MODULE_DIR$/build" />
+      <excludeFolder url="file://$MODULE_DIR$/out" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

MetricsReloaded/src/main/java/com/sixrr/metrics/metricModel/MetricsExecutionContextImpl.java

@@ -79,7 +79,8 @@ public void onCancel() {
 
     public void calculateMetrics(MetricsProfile profile, final MetricsResultsHolder resultsHolder) {
         final ProgressIndicator indicator;
-        if (ApplicationManager.getApplication().isUnitTestMode()) {
+
+        if (ApplicationManager.getApplication().isUnitTestMode() || ApplicationManager.getApplication().isHeadlessEnvironment()) {
             indicator = new EmptyProgressIndicator();
         } else {
             indicator = ProgressManager.getInstance().getProgressIndicator();

MetricsReloaded/utils/src/main/java/com/sixrr/metrics/utils/MethodUtils.java

@@ -26,6 +26,7 @@
 import org.jetbrains.annotations.Nullable;
 
 import java.util.*;
+import java.util.regex.Pattern;
 
 public final class MethodUtils {
 
@@ -64,6 +65,14 @@ public static boolean isStatic(PsiModifierListOwner unit) {
         return unit.hasModifierProperty(PsiModifier.STATIC);
     }
 
+    public static boolean isSynchronized(PsiModifierListOwner unit) {
+        return unit.hasModifierProperty(PsiModifier.SYNCHRONIZED);
+    }
+
+    public static boolean isOverriding(PsiMethod method) {
+        return method.findSuperMethods().length != 0;
+    }
+
     public static boolean isPrivate(PsiModifierListOwner unit) {
         return unit.hasModifierProperty(PsiModifier.PRIVATE);
     }
@@ -223,8 +232,24 @@ private static boolean hasField(@NotNull final PsiClass aClass,
         return false;
     }
 
-    public static boolean isOverriding(PsiMethod method) {
-        return method.findSuperMethods().length != 0;
+    public static boolean isPublic(PsiMethod method) {
+        if (method.hasModifierProperty(PsiModifier.PUBLIC)) {
+            return true;
+        }
+
+        PsiClass containingClass = method.getContainingClass();
+
+        return containingClass != null && containingClass.isInterface();
+    }
+
+    public static String extractMethodDeclaration(final @NotNull PsiMethod method) {
+        String code = method.getText();
+
+        code = Pattern.compile("/\\*.*?\\*/", Pattern.DOTALL).matcher(code).replaceAll("");
+        code = Pattern.compile("//.*?$", Pattern.DOTALL | Pattern.MULTILINE).matcher(code).replaceAll("");
+
+        code = Pattern.compile("\\{.*}", Pattern.DOTALL).matcher(code).replaceAll("");
+        return code.trim();
     }
 
     /**

MetricsReloaded/utils/src/main/java/com/sixrr/metrics/utils/ProjectUtils.java

@@ -0,0 +1,34 @@
+package com.sixrr.metrics.utils;
+
+import com.intellij.ide.impl.ProjectUtil;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.project.ProjectManager;
+import com.intellij.openapi.roots.ProjectRootManager;
+import com.intellij.openapi.util.Disposer;
+import org.jetbrains.annotations.Nullable;
+
+import java.nio.file.Path;
+
+public class ProjectUtils {
+    private ProjectUtils() {
+    }
+
+    @Nullable
+    public static Project loadProjectWithAllDependencies(Path projectPath) {
+        for (Project openedProject : ProjectManager.getInstance().getOpenProjects()) {
+            if (!ProjectManager.getInstance().closeProject(openedProject)) {
+                throw new IllegalStateException("Cannot close previous project");
+            }
+            Disposer.dispose(openedProject);
+            if (!openedProject.isDisposed()) {
+                throw new IllegalStateException("Cannot dispose project");
+            }
+        }
+        final Project project = ProjectUtil.openOrImport(projectPath.toAbsolutePath().toString(),
+                null, false);
+        return project == null ||
+                ProjectRootManager.getInstance(project).getProjectSdk() == null ||
+                !project.isInitialized() ?
+                null : project;
+    }
+}

algorithms-evaluation/build.gradle

@@ -0,0 +1,12 @@
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    compile project(':core')
+    compile project(':features-extraction')
+    // compile project(':utils')
+
+    // testCompile project(':utils').sourceSets.test.output
+    testCompile group: 'org.mockito', name: 'mockito-core', version: '2.19.1'
+}

algorithms-evaluation/src/main/java/org/jetbrains/research/groups/ml_methods/evaluation/AbstractEvaluationResult.java

@@ -0,0 +1,76 @@
+package org.jetbrains.research.groups.ml_methods.evaluation;
+
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.research.groups.ml_methods.algorithm.Algorithm;
+
+import java.util.EnumMap;
+import java.util.function.Supplier;
+
+import static java.lang.Math.pow;
+
+public abstract class AbstractEvaluationResult implements EvaluationResult {
+    private final @NotNull EnumMap<Evaluation, Supplier<Double>> evaluationFunctions =
+            new EnumMap<>(Evaluation.class);
+    private final @NotNull Algorithm evaluatingAlgorithm;
+
+    {
+        evaluationFunctions.put(Evaluation.GOOD_PRECISION, this::getGoodPrecision);
+        evaluationFunctions.put(Evaluation.GOOD_RECALL, this::getGoodRecall);
+        evaluationFunctions.put(Evaluation.BAD_PRECISION, this::getBadPrecision);
+        evaluationFunctions.put(Evaluation.BAD_RECALL, this::getBadRecall);
+        evaluationFunctions.put(Evaluation.MSE, this::getMSE);
+        evaluationFunctions.put(Evaluation.ME, this::getME);
+    }
+
+    AbstractEvaluationResult(@NotNull Algorithm evaluatingAlgorithm) {
+        this.evaluatingAlgorithm = evaluatingAlgorithm;
+    }
+
+    @Override
+    public Algorithm getAlgorithm() {
+        return evaluatingAlgorithm;
+    }
+
+    @Override
+    public double getGoodPrecision() {
+        return (double) getNumberOfFoundGood() / getNumberOfFoundBadAndGood();
+    }
+
+    @Override
+    public double getGoodRecall() {
+        return (double) getNumberOfFoundGood() / getNumberOfGood();
+    }
+
+    @Override
+    public double getBadPrecision() {
+        return (double) (getNumberOfBad() - getNumberOfFoundBad()) / (getNumberOfBadAndGood() - getNumberOfFoundBadAndGood());
+    }
+
+    @Override
+    public double getBadRecall() {
+        return (double) (getNumberOfBad() - getNumberOfFoundBad()) / getNumberOfBad();
+    }
+
+    @Override
+    public double getMSE() {
+        return getErrors().stream().mapToDouble(Double::doubleValue).map(operand -> pow(operand, 2)).sum() / getErrors().size();
+    }
+
+    @Override
+    public double getME() {
+        return getErrors().stream().mapToDouble(Double::doubleValue).sum() / getErrors().size();
+    }
+
+    private int getNumberOfBadAndGood() {
+        return getNumberOfBad() + getNumberOfGood();
+    }
+
+    private int getNumberOfFoundBadAndGood() {
+        return getNumberOfFoundBad() + getNumberOfFoundGood();
+    }
+
+    @Override
+    public Supplier<Double> getEvaluation(Evaluation evaluation) {
+        return evaluationFunctions.get(evaluation);
+    }
+}

algorithms-evaluation/src/main/java/org/jetbrains/research/groups/ml_methods/evaluation/AlgorithmEvaluator.java

@@ -0,0 +1,45 @@
+package org.jetbrains.research.groups.ml_methods.evaluation;
+
+import com.intellij.analysis.AnalysisScope;
+import com.intellij.openapi.project.Project;
+import com.sixrr.metrics.profile.MetricsProfile;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.research.groups.ml_methods.algorithm.Algorithm;
+import org.jetbrains.research.groups.ml_methods.refactoring.CalculatedRefactoring;
+import org.jetbrains.research.groups.ml_methods.refactoring.MoveToClassRefactoring;
+import org.jetbrains.research.groups.ml_methods.algorithm.RefactoringExecutionContext;
+import org.jetbrains.research.groups.ml_methods.utils.MetricsProfilesUtil;
+
+import java.util.Collections;
+import java.util.List;
+
+public class AlgorithmEvaluator {
+    @NotNull
+    static EvaluationResult evaluate(@NotNull AnalysisScope scope, @NotNull Algorithm algorithm,
+                                            @NotNull List<MoveToClassRefactoring> good, @NotNull List<MoveToClassRefactoring> bad) {
+        MetricsProfile profile =
+                MetricsProfilesUtil.createProfile("evaluation_profile", algorithm.requiredMetrics());
+        RefactoringExecutionContext context = new RefactoringExecutionContext(scope.getProject(), scope, profile,
+                Collections.singletonList(algorithm), false, null);
+
+        context.executeSynchronously();
+
+        List<CalculatedRefactoring> foundRefactorings = context.getAlgorithmResults().get(0).getRefactorings();
+        return new ProjectEvaluationResult(foundRefactorings, good, bad, algorithm);
+    }
+
+    @NotNull
+    static EvaluationResult evaluate(@NotNull Project project, @NotNull Algorithm algorithm,
+                                            @NotNull List<MoveToClassRefactoring> good, @NotNull List<MoveToClassRefactoring> bad) {
+        return evaluate(new AnalysisScope(project), algorithm, good, bad);
+    }
+
+    @NotNull
+    static EvaluationResult evaluate(@NotNull ProjectToEvaluate projectToEvaluate,
+                                     @NotNull Algorithm algorithm) {
+        return evaluate(projectToEvaluate.getProject(),
+                algorithm,
+                projectToEvaluate.getGoodRefactorings(),
+                projectToEvaluate.getBadRefactorings());
+    }
+}

algorithms-evaluation/src/main/java/org/jetbrains/research/groups/ml_methods/evaluation/AlgorithmsEvaluationApplicationStarter.java

@@ -0,0 +1,99 @@
+package org.jetbrains.research.groups.ml_methods.evaluation;
+
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.application.ApplicationStarter;
+import com.intellij.openapi.application.ex.ApplicationEx;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.research.groups.ml_methods.algorithm.Algorithm;
+import org.jetbrains.research.groups.ml_methods.algorithm.AlgorithmsRepository;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+
+public class AlgorithmsEvaluationApplicationStarter implements ApplicationStarter {
+    private static final ApplicationEx APPLICATION = (ApplicationEx) ApplicationManager.getApplication();
+    private static final int NUMBER_OF_ARGUMENTS = 4;
+    private static final @NotNull Logger LOGGER =
+            Logger.getLogger(AlgorithmsEvaluationApplicationStarter.class);
+
+    static {
+        LOGGER.setLevel(Level.INFO);
+        LOGGER.addAppender(new ConsoleAppender(new PatternLayout("%p %m%n")));
+    }
+
+    private static void checkCommandLineArguments(@NotNull String[] args) {
+        if (args.length != NUMBER_OF_ARGUMENTS) {
+            printUsage();
+            APPLICATION.exit(true, true);
+        }
+    }
+
+    private static void printUsage() {
+        System.out.println("Usage: algorithms-evaluation <path to dataset folder> <algorithms names> <pathToSaveResults>");
+    }
+
+    @Override
+    public String getCommandName() {
+        return "algorithms-evaluation";
+    }
+
+    @Override
+    public void premain(String[] args) {
+    }
+
+    public void main(String[] args) {
+        try {
+            checkCommandLineArguments(args);
+            Path datasetPath = Paths.get(args[1]);
+            List<String> algorithmsNames = Arrays.asList(args[2].split(","));
+            Path pathToSaveResults = Paths.get(args[3]);
+            List<Algorithm> algorithmsToEvaluate;
+            if (algorithmsNames.get(0).equals("")) {
+                algorithmsToEvaluate = AlgorithmsRepository.getAvailableAlgorithms();
+            } else {
+                algorithmsToEvaluate = algorithmsNames.stream()
+                        .map(algorithmName -> AlgorithmsRepository.getAlgorithmByName(algorithmName)
+                                .orElseThrow(() -> new IllegalArgumentException("No such algorithm")))
+                        .collect(Collectors.toList());
+            }
+            List<EvaluationResult> evaluationResults = new ArrayList<>();
+            for (Algorithm algorithm : algorithmsToEvaluate) {
+                evaluationResults.add(JBAlgorithmEvaluator.evaluateDataset(datasetPath, algorithm));
+            }
+            evaluationResults.forEach(this::printEvaluationResult);
+            if (!pathToSaveResults.toString().equals("")) {
+                EvaluationResultsWriter.writeTable(evaluationResults, pathToSaveResults);
+            }
+        } catch (Throwable throwable) {
+            System.out.println(throwable.getClass().getSimpleName() + ": " + throwable.getMessage());
+            throwable.printStackTrace();
+        } finally {
+            APPLICATION.exit(true, true);
+        }
+    }
+
+    private void printEvaluationResult(EvaluationResult evaluationResult) {
+        System.out.println("==================");
+        System.out.println("EVALUATION RESULT FOR " + evaluationResult.getAlgorithm().getDescriptionString());
+        System.out.println("Number of good: " + evaluationResult.getNumberOfGood());
+        System.out.println("Number of found good: " + evaluationResult.getNumberOfFoundGood());
+        System.out.println("Number of bad: " + evaluationResult.getNumberOfBad());
+        System.out.println("Number of found bad: " + evaluationResult.getNumberOfFoundBad());
+        System.out.println("Number of found others: " + evaluationResult.getNumberOfFoundOthers());
+        System.out.println("Good precision: " + evaluationResult.getGoodPrecision());
+        System.out.println("Bad precision: " + evaluationResult.getBadPrecision());
+        System.out.println("Good recall: " + evaluationResult.getGoodRecall());
+        System.out.println("Bad recall: " + evaluationResult.getBadRecall());
+        System.out.println("Mean squared error (MSE): " + evaluationResult.getMSE());
+        System.out.println("Mean error (ME): " + evaluationResult.getME());
+    }
+}