upload scripts

scripts/clone.sh

@@ -0,0 +1,31 @@
+input=$1 #project_url
+cloneDir=$2 # dir to clone all projects
+
+timeStamp=$(echo -n $(date "+%Y-%m-%d %H:%M:%S") | shasum | cut -f 1 -d " ")
+
+mkdir -p ${cloneDir}
+mkdir -p ./output/$timeStamp/logs
+
+mainDir=$(pwd)/${cloneDir}
+logDir=$(pwd)/output/$timeStamp/logs
+
+exec 3>&1 4>&2
+trap $(exec 2>&4 1>&3) 0 1 2 3
+exec 1>$logDir/$timeStamp.log 2>&1
+
+for info in $(cat $input); do
+    url=$(echo $info | cut -d, -f1)
+    sha=$(echo $info | cut -d, -f2)
+    module=$(echo $info | cut -d, -f3)
+    project=${url##*/}
+
+    cd ${mainDir}
+    mkdir -p $sha
+
+    cd ${sha}
+    git clone ${url}
+    cd ${project}
+    git checkout ${sha}
+    cd ../../..
+
+    done

scripts/extract_fixes.py

@@ -0,0 +1,169 @@
+import csv
+import sys
+import os
+import git
+import get_uniq_projects
+import openai
+import datetime
+import glob
+import utils
+import tiktoken
+import subprocess
+import javalang
+from subprocess import Popen, PIPE
+import re
+
+run_test_cmds = "/home/azureuser/flaky/cmds/run_test.sh"
+run_nondex_cmds = "/home/azureuser/flaky/cmds/run_nondex.sh"
+run_surefire_cmds = "/home/azureuser/flaky/cmds/run_surefire.sh"
+checkout_project_cmds = "/home/azureuser/flaky/cmds/checkout_project.sh"
+restore_project_cmds = "/home/azureuser/flaky/cmds/stash_project.sh"
+
+def match_import(response):
+    print(response)
+    # import_pattern = re.compile(r'^\s*import\s+([\w.]+);', re.MULTILINE)
+    # import_pattern = re.compile(r'^\s*import\s+([\w.]+);', re.MULTILINE)
+    # import_pattern = re.compile(r'import\s+([\w.]+);')
+    import_pattern = re.compile(r'^\s*import\s+([\w.]+);', re.MULTILINE)
+    # import_pattern = re.compile(r'^(import\s+[\w.]+;)', re.MULTILINE)
+    imp_matches = import_pattern.findall(response)
+    print(imp_matches)
+
+def git_stash(project, sha, cloneDir,file_path):
+    result = subprocess.run(["bash",checkout_project_cmds,project,sha,cloneDir,file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output = result.stdout.decode('utf-8')
+    print(output,flush = True)
+
+def restore_project(project, sha, cloneDir):
+    result = subprocess.run(["bash",restore_project_cmds,project,sha,cloneDir], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output = result.stdout.decode('utf-8')
+    print(output,flush = True)
+
+def match_patch(response):
+    if ("<class changed>") in response:
+        if ("}") in response:
+            right_idx = response.rindex("}")
+            if ("package ") in response: # whole class file in the answer
+                left_idx = response.index("package ")
+                potential_patch = response[left_idx:right_idx+1]
+                return potential_patch,0
+            else: # only test method in the answer
+                if ("@Test") in response:
+                    left_idx = response.index("@Test")
+                    potential_patch = response[left_idx:right_idx+1]
+                    return potential_patch,1
+                if ("public void ") in response:
+                    left_idx = response.index("public void ")
+                    potential_patch = response[left_idx:right_idx+1]
+                    return potential_patch,1
+
+    if ("<method changed>") in response:
+        if ("}") in response:
+            right_idx = response.rindex("}")
+            if ("@Test") in response:
+                left_idx = response.index("@Test")
+                potential_patch = response[left_idx:right_idx+1]
+                return potential_patch,1
+            if ("public void ") in response:
+                left_idx = response.index("public void ")
+                potential_patch = response[left_idx:right_idx+1]
+                return potential_patch,1
+    return None,None
+
+def replace_last(source_string, replace_what, replace_with):
+    head, _sep, tail = source_string.rpartition(replace_what)
+    return head + replace_with + tail
+
+def apply_patch(project,sha,module,test,test_type,method_name,patch,patch_type,file_path,cloneDir):
+    format_test = replace_last(test, '.', '#')
+
+    if patch == None:
+        print("[ERROR]No Patch",flush = True)
+        return None
+    try:
+        if patch_type == 1: #only change method
+            file = open(file_path, 'r', errors='ignore')
+            class_content = file.read()
+            res = utils.get_test_method(method_name, class_content) #res = [start,end,method_name,method_code,node.annotations]
+            if res == None:
+                return None
+            method_code = res[3]
+            annotations = res[4]
+            new_patch = patch
+            for annotation in annotations:
+                if "@"+annotation.name in patch:
+                    new_patch = patch.replace("@" + annotation.name, "")
+            fixed_class = class_content.replace(method_code,new_patch)
+
+
+            print(("[Before fix] Running test {} with type {} from project {} sha {} module {} \
+                ").format(format_test, test_type, project, sha, module),flush = True)
+            git_stash(project, sha, cloneDir,file_path)
+            verify_by_tool(test_type,project,sha,format_test,module,cloneDir,"BeforeFix")
+
+            print(("[Applying FIX] Applying patch on test {}").format(format_test),flush = True)
+            f = open(file_path, "w", errors='ignore')
+            f.write(fixed_class)
+            f.close()
+
+            print(("[After fix] Running test {} with type {} from project {} sha {} module {} \
+                    ").format(format_test, test_type, project, sha, module),flush = True)
+            verify_by_tool(test_type,project,sha,format_test,module,cloneDir,"AfterFix")
+            git_stash(project, sha, cloneDir,file_path)
+
+            return new_patch
+
+        if patch_type == 0:
+
+            print(("[Before fix] Running test {} with type {} from project {} sha {} module {} \
+                ").format(format_test, test_type, project, sha, module),flush = True)
+            git_stash(project, sha, cloneDir,file_path)
+            verify_by_tool(test_type,project,sha,format_test,module,cloneDir,"BeforeFix")
+
+            print(("[Applying FIX] Applying patch on test {}\n").format(format_test),flush = True)
+
+            f = open(file_path, "w", errors='ignore')
+            f.write(patch)
+            f.close()
+
+            print(("[After fix] Running test {} with type {} from project {} sha {} module {} \
+                    ").format(format_test, test_type, project, sha, module),flush = True)
+            verify_by_tool(test_type,project,sha,format_test,module,cloneDir,"AfterFix")
+            git_stash(project, sha, cloneDir,file_path)
+            # exit(0)
+            return patch
+    except:
+        return None
+
+def verify_by_tool(test_type,project,sha,format_test,module,cloneDir,tag,times):
+    if test_type == "ID" or test_type == "NOD":
+        print(("RUNNING NonDex {} time(s) on test {} with type {} from project {} sha {} module {} \
+              ").format(times, format_test, test_type, project, sha, module),flush = True)
+        output = run_nondex(project,sha,format_test,module,cloneDir,tag,times)
+        return output
+    if test_type == "Brit":
+        print(("RUNNING NonDex {} time(s) on test {} with type Brit from project {} sha {} module {} \
+              ").format(times, format_test, test_type, project, sha, module),flush = True)
+        output = run_nondex(project,sha,format_test,module,cloneDir,tag,times)
+        return output
+
+
+#(test_type,project,sha,module,polluter_format_test,victim_format_test,cloneDir,tag,times,victim_file_path)
+def verify_by_surefire(test_type,project,sha,module,polluter_format_test,victim_format_test,cloneDir,tag,times): 
+    if "OD" in test_type:
+        print(("RUNNING Surefire {} time(s) on polluter {} and victim {} with type {} from project {} sha {} module {} \
+              ").format(times,polluter_format_test,victim_format_test, test_type, project, sha, module),flush = True)
+        output = run_surefire(project,sha,module,polluter_format_test,victim_format_test,cloneDir,tag,times)
+        return output
+
+def run_nondex(project,sha,test,module,cloneDir,tag,times):
+    result = subprocess.run(["bash",run_nondex_cmds,project,sha,test,module,cloneDir,tag,times], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output = result.stdout.decode('utf-8')
+    print(output,flush = True)
+    return output
+
+def run_surefire(project,sha,module,polluter_format_test,victim_format_test,cloneDir,tag,times):
+    result = subprocess.run(["bash",run_surefire_cmds,project,sha,module,polluter_format_test,victim_format_test,cloneDir,tag,times], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    output = result.stdout.decode('utf-8')
+    print(output,flush = True)
+    return output

scripts/install.sh

@@ -0,0 +1,64 @@
+input=$1 #project_url
+cloneDir=$2 # dir to clone all projects
+
+timeStamp=$(echo -n $(date "+%Y-%m-%d %H:%M:%S") | shasum | cut -f 1 -d " ")
+
+mkdir -p ./output/$timeStamp/install_logs
+
+mainDir=$(pwd)/${cloneDir}
+logDir=$(pwd)/output/$timeStamp/install_logs
+
+exec 3>&1 4>&2
+trap $(exec 2>&4 1>&3) 0 1 2 3
+exec 1>$logDir/$timeStamp.log 2>&1
+
+install(){
+    t=$(echo -n $(date "+%Y-%m-%d %H:%M:%S") | shasum | cut -f 1 -d " ")
+    mvn install -pl ${module} -am -DskipTests -Dfindbugs.skip=true -Dbasepom.check.skip-prettier -Dgpg.skip -Drat.skip -Dskip.npm -Dskip.yarn -Dskip.bower -Dskip.grunt -Dskip.gulp -Dskip.jspm -Dskip.karma -Dskip.webpack -Dcheckstyle.skip -Denforcer.skip=true -Dspotbugs.skip -Dmaven.test.failure.ignore=true -Djacoco.skip -Danimal.sniffer.skip -Dmaven.antrun.skip -Dfmt.skip -Dskip.npm -Dlicense.skipCheckLicense -Dlicense.skipAddThirdParty=true -Dfindbugs.skip -Dlicense.skip -DskipDockerBuild -DskipDockerTag -DskipDockerPush -DskipDocker -Denforcer.skip |&tee install${project}${t}.log
+    res="$(grep 'BUILD ' ${mainDir}/${sha}/${project}/install${project}${t}.log)"
+    echo "build-result:" ${project} ${sha} ${module} ${res}
+    #mvn install -pl ${module} -am -DskipTests --log-file install_${project}.log
+}
+
+for info in $(cat $input); do
+    url=$(echo $info | cut -d, -f1)
+    sha=$(echo $info | cut -d, -f2)
+    module=$(echo $info | cut -d, -f3)
+    project=${url##*/}
+
+    cd ${mainDir}/${sha}/${project}
+    echo ${mainDir}/${sha}/${project}
+    echo "run git stash"
+    git stash
+    java_version="$(grep "<jdk.*>" pom.xml)"
+    if [[ $java_version == *"11"* ]]; then
+        echo "java version 11"
+        export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64
+        export PATH=$JAVA_HOME/bin:$PATH
+    elif [[ $java_version == *"8"* ]]; then
+        echo "java version 8"
+        export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
+        export PATH=$JAVA_HOME/bin:$PATH
+    elif [[ $java_version == *"1.8"* ]]; then
+        echo "java version 8"
+        export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
+        export PATH=$JAVA_HOME/bin:$PATH
+    elif [[ $url == "https://github.com/INRIA/spoon" ]]; then
+        echo "java version 11"
+        export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64
+        export PATH=$JAVA_HOME/bin:$PATH
+    elif [[ $project == "cloudstack" ]]; then
+        echo "java version 11"
+        export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64
+        export PATH=$JAVA_HOME/bin:$PATH
+    else
+        echo "java version not found, will use 8" $java_version
+        export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64
+        export PATH=$JAVA_HOME/bin:$PATH
+    fi
+
+
+    install
+
+    cd ../../..
+    done

scripts/setup.sh

@@ -0,0 +1,3 @@
+#python3.8.10
+pip3 install git+https://github.com/jose/javalang.git@start_position_and_end_position
+pip3 install gitpython

scripts/update_pom.py

@@ -0,0 +1,42 @@
+import os
+import sys
+
+def dependency_exists(pom_lines, artifactId):
+    for line in pom_lines:
+        if f'<artifactId>{artifactId}</artifactId>' in line:
+            return True
+    return False
+
+def add_dependency(pom_path, dependency_lines):
+
+    updated_pom_lines = []
+    d_list = dependency_lines.split("</dependency>")
+    for dep in d_list:
+        if "<dependency>" in dep:
+            d = "<dependency>\n" + dep.split("<dependency>")[1]
+            if "<artifactId>" in d:
+                with open(pom_path, 'r') as pom_file:
+                    pom_lines = pom_file.readlines()
+                artifactId = (d.split("<artifactId>")[1]).split("</artifactId>")[0]
+                if "mycompany" in artifactId:
+                    print("will not add", artifactId)
+                if "artifact" in artifactId:
+                    print("will not add", artifactId)
+                if dependency_exists(pom_lines, artifactId):
+                    print(artifactId, " already in pom.xml, no need to add")
+                else:
+                    print(artifactId, " will be added")
+                    for line in pom_lines:
+                        updated_pom_lines.append(line)
+                        if '<dependencies>' in line:
+                            updated_pom_lines.extend(d + "</dependency>\n")
+                    with open(pom_path, 'w') as pom_file_new:
+                        pom_file_new.writelines(updated_pom_lines)
+
+if __name__ == "__main__":
+    # args = sys.argv[1:]
+    # pom_path = args[0]
+    # dependency_lines = args[1]
+    # path= "/home/azureuser/flaky/projects/6fb5cd049b57a22d2ec4465d204c15f1c90dd325/adyen-java-api-library/pom.xml"
+    # toadd = '\n<dependency>\n    <groupId>com.fasterxml.jackson.core</groupId>\n    <artifactId>jackson-databind22</artifactId>\n    <version>2.12.3</version>\n</dependency>\n'
+    # add_dependency(path,toadd + toadd)