Skip to content

Commit 7a894cd

Browse files
izosakizosak
authored
Add files via upload
1 parent b1172d3 commit 7a894cd

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+2655
-0
lines changed

‎TSExtractor/TSExtractor/Common/CommandLineValues.ts

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
export class CommandLineValues {
2+
public File: string;
3+
public Dir:string;
4+
public MaxPathLength:number;
5+
public MaxPathWidth:number;
6+
public NoHash:boolean;
7+
public NumThreads:number;
8+
public MinCodeLength:number;
9+
public MaxCodeLength:number;
10+
public PrettyPrint:boolean;
11+
public MaxChildId:number;
12+
public debugMode:boolean;
13+
public testSetMode:boolean;
14+
public logDir:string
15+
16+
constructor() {
17+
this.setArgsDeafultValues();
18+
var commandLineArgs = this.getCommandLineArgs();
19+
this.updateCommanLineArgs(commandLineArgs);
20+
}
21+
22+
private setArgsDeafultValues() {
23+
this.File = null;
24+
this.Dir = null;
25+
this.MaxPathLength = 100;
26+
this.MaxPathWidth = 10;
27+
this.NoHash = false;
28+
this.NumThreads = 32;
29+
this.MinCodeLength = 1;
30+
this.MaxCodeLength = 10000;
31+
this.PrettyPrint = false;
32+
this.MaxChildId = Number.MAX_SAFE_INTEGER;
33+
this.debugMode = false;
34+
this.testSetMode = false;
35+
this.logDir = "TSE_log/";
36+
}
37+
38+
private getCommandLineArgs() {
39+
var optionDefinitions = [
40+
{ name: 'dir', type: String},
41+
{ name: 'file', type: String },
42+
{ name: 'max_path_length', type: Number },
43+
{ name: 'max_path_width', type: Number },
44+
{ name: 'no_hash', type: Boolean },
45+
{ name: 'num_threads', type: Number },
46+
{ name: 'min_code_len', type: Number },
47+
{ name: 'max_code_len', type: Number },
48+
{ name: 'pretty_print', type: Boolean },
49+
{ name: 'max_child_id', type: Number },
50+
{ name: 'debug', type: Number },
51+
{ name: 'test_set', type: Number },
52+
{ name: 'log_dir', type: String}
53+
];
54+
const commandLineArgs = require('command-line-args');
55+
const options = commandLineArgs(optionDefinitions);
56+
return options;
57+
}
58+
59+
private updateCommanLineArgs(commandLineArgs : any) : void {
60+
if(commandLineArgs.max_path_length){
61+
this.MaxPathLength=commandLineArgs.max_path_length;
62+
}
63+
if(commandLineArgs.max_path_width){
64+
this.MaxPathWidth=commandLineArgs.max_path_width;
65+
}
66+
if(commandLineArgs.file){
67+
this.File=commandLineArgs.file;
68+
}
69+
if(commandLineArgs.dir){
70+
this.Dir=commandLineArgs.dir;
71+
}
72+
if(commandLineArgs.no_hash){
73+
this.NoHash=commandLineArgs.no_hash;
74+
}
75+
if(commandLineArgs.num_threads){
76+
this.NumThreads=commandLineArgs.num_threads;
77+
}
78+
if(commandLineArgs.min_code_len){
79+
this.MinCodeLength=commandLineArgs.min_code_len;
80+
}
81+
if(commandLineArgs.max_code_len){
82+
this.MaxCodeLength=commandLineArgs.max_code_len;
83+
}
84+
if(commandLineArgs.pretty_print){
85+
this.PrettyPrint=commandLineArgs.pretty_print;
86+
}
87+
if(commandLineArgs.max_child_id){
88+
this.MaxChildId=commandLineArgs.max_child_id;
89+
}
90+
if(commandLineArgs.debug) {
91+
this.debugMode=commandLineArgs.debug;
92+
}
93+
if(commandLineArgs.test_set) {
94+
this.testSetMode=commandLineArgs.test_set;
95+
}
96+
if(commandLineArgs.log_dir) {
97+
let log_dir: string = commandLineArgs.log_dir;
98+
if (!commandLineArgs.log_dir.endsWith("/")) log_dir += "/";
99+
this.logDir=log_dir;
100+
}
101+
}
102+
}

‎TSExtractor/TSExtractor/Common/Common.ts

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
2+
export class Common{
3+
4+
public static EmptyString: string="";
5+
public static UTF8:string = "utf-8";
6+
public static internalSeparator:string = "|";
7+
public static EvaluateTempDir:string = "EvalTemp";
8+
public static FieldAccessExpr:string = "FieldAccessExpr";
9+
public static ClassOrInterfaceType:string = "ClassOrInterfaceType";
10+
public static MethodDeclaration:string = "MethodDeclaration";
11+
public static NameExpr:string = "NameExpr";
12+
public static MethodCallExpr:string = "MethodCallExpr";
13+
public static DummyNode:string = "DummyNode";
14+
public static BlankWord:string = "BLANK"
15+
public static c_MaxLabelLength:number = 50;
16+
public static methodName:string = "METHOD_NAME";
17+
18+
public static normalizeName(original: string ,defaultString: string): string {
19+
let carefulStripped = original.toLowerCase()
20+
.replace(/\s/g, "") // whitespaces
21+
.replace(/[\"',]/g, "") // quotes, apostrophies, commas
22+
.replace(/\\P{Print}/g, ""); // unicode weird characters
23+
let stripped: string = original.replace(/[^A-Za-z]/g, "");
24+
if (stripped.length == 0) {
25+
if (carefulStripped.length == 0) {
26+
return defaultString;
27+
} else {
28+
return carefulStripped;
29+
}
30+
} else {
31+
return stripped;
32+
}
33+
}
34+
35+
public static splitToSubtokens(str1: string): Array<string> {
36+
let str2: string = str1.trim();
37+
let strArray = str2.split("(?<=[a-z])(?=[A-Z])|_|[0-9]|(?<=[A-Z])(?=[A-Z][a-z])|\\s+");
38+
return strArray.filter(s => s.length > 0).map(s => Common.normalizeName(s, ""))
39+
.filter(s => s.length > 0);
40+
}
41+
}

‎TSExtractor/TSExtractor/Common/utilities.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
export function hashCode(str: string) : number{
2+
var hash = 0;
3+
for (let i = 0; i < str.length; i++) {
4+
let chr = str.charCodeAt(i);
5+
hash = ((hash << 5) - hash) + chr;
6+
hash |= 0; // Convert to 32bit integer
7+
}
8+
return hash;
9+
};

‎TSExtractor/TSExtractor/ExtractFeaturesTask.ts

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
import { CommandLineValues } from './Common/CommandLineValues';
2+
import { ProgramFeatures } from './FeaturesEntities/ProgramFeatures';
3+
import { Common } from './Common/Common';
4+
import { FeatureExtractor } from './FeaturesExtractor';
5+
import { appendFileSync, writeFileSync } from 'fs';
6+
import fs = require('fs');
7+
import path = require('path');
8+
9+
export class ExtractFeaturesTask {
10+
static log_file: string = "TSE.log";
11+
static m_CommandLineValues: CommandLineValues;
12+
13+
/**
14+
* Processes the command line values and returns the number of extracted files.
15+
*/
16+
public static process(): number {
17+
ExtractFeaturesTask.log_file = this.m_CommandLineValues.logDir + ExtractFeaturesTask.log_file;
18+
let num_extracted: number = 0;
19+
if (this.m_CommandLineValues.File != null) {
20+
let extracted: boolean = ExtractFeaturesTask.processFile(this.m_CommandLineValues.File);
21+
if (extracted) num_extracted = 1;
22+
} else if (this.m_CommandLineValues.Dir != null) {
23+
num_extracted = ExtractFeaturesTask.processDir();
24+
}
25+
return num_extracted;
26+
}
27+
28+
/**
29+
* Processes a file and returns true if it was extracted (meaning, if there were program features extracted out of it)
30+
* and false otherwise.
31+
* @param filePath - the file to be processed.
32+
*/
33+
private static processFile(filePath: string): boolean {
34+
let processing_msg: string = new Date().toLocaleString() + ": Processing file:\n\t\t\t\t\t\t" + filePath + "\n";
35+
appendFileSync(ExtractFeaturesTask.log_file, processing_msg, Common.UTF8);
36+
if (!filePath.endsWith('.ts')) {
37+
let processing_msg: string = new Date().toLocaleString() + ": Given file is not a .ts file:\n\t\t\t\t\t\t" + filePath + "\n";
38+
appendFileSync(ExtractFeaturesTask.log_file, processing_msg, Common.UTF8);
39+
return false;
40+
}
41+
try {
42+
var featureExtractor: FeatureExtractor = new FeatureExtractor(this.m_CommandLineValues, filePath);
43+
// var features: Array<ProgramFeatures> = featureExtractor.extractFeatures();
44+
var features: Array<ProgramFeatures> = featureExtractor.extractFeatures();
45+
if (features == null || features.length == 0) {
46+
let no_paths_msg: string = new Date().toLocaleString() + ": No paths in file:\n\t\t\t\t\t\t" + filePath + "\n";
47+
appendFileSync(ExtractFeaturesTask.log_file, no_paths_msg, Common.UTF8);
48+
return false;
49+
}
50+
var toPrint: Array<string> = ExtractFeaturesTask.featuresToString(features);
51+
for (let i = 0; i < toPrint.length; i++) {
52+
console.log(toPrint[i]);
53+
}
54+
}
55+
catch(e) {
56+
let e_msg: string = new Date().toLocaleString() + ": Exception in file:\n\t\t\t\t\t\t" + filePath + ": \"" + e + "\"\n";
57+
appendFileSync(ExtractFeaturesTask.log_file, e_msg, Common.UTF8);
58+
return false;
59+
}
60+
let finished_msg: string = new Date().toLocaleString() + ": Finished processing file:\n\t\t\t\t\t\t" + filePath + "\n";
61+
appendFileSync(ExtractFeaturesTask.log_file, finished_msg, Common.UTF8);
62+
return true;
63+
}
64+
65+
/**
66+
* Processes a directory and returns the number of .ts files that were extracted.
67+
*/
68+
private static processDir(): number {
69+
let processing_msg: string = new Date().toLocaleString() + ": Processing directory:\n\t\t\t\t\t\t" + this.m_CommandLineValues.Dir + "\n";
70+
appendFileSync(ExtractFeaturesTask.log_file, processing_msg, Common.UTF8);
71+
let dir_path: string = this.m_CommandLineValues.Dir;
72+
var num_ts_files: number = 0, num_processed: number = 0, num_extracted: number = 0;
73+
let dir_path_splitted: Array<string> = dir_path.split("/");
74+
let dir_name: string = dir_path_splitted[dir_path_splitted.length-1];
75+
let dir_progress_file: string = this.m_CommandLineValues.logDir + dir_name + "_progress.log";
76+
77+
// count the number of .ts files
78+
walkDir(dir_path, function(filePath: string) {
79+
if (filePath.endsWith('.ts')) {
80+
num_ts_files++;
81+
}
82+
});
83+
84+
let start_time: Date = new Date();
85+
let progress_msg: string =
86+
ExtractFeaturesTask.getProgressMessage(num_processed, num_extracted, num_ts_files, dir_path, start_time, false);
87+
writeFileSync(dir_progress_file, progress_msg);
88+
89+
// extract all .ts files in the directory
90+
walkDir(dir_path, function(filePath: string) {
91+
if (!filePath.endsWith('.ts')) {
92+
return;
93+
}
94+
let extracted: boolean = ExtractFeaturesTask.processFile(filePath);
95+
num_processed++;
96+
if (extracted) num_extracted++;
97+
if (num_processed%5 == 0 || num_processed == num_ts_files) {
98+
let progress_msg: string =
99+
ExtractFeaturesTask.getProgressMessage(num_processed, num_extracted, num_ts_files, dir_path, start_time, false);
100+
writeFileSync(dir_progress_file, progress_msg);
101+
}
102+
});
103+
104+
progress_msg =
105+
ExtractFeaturesTask.getProgressMessage(num_processed, num_extracted, num_ts_files, dir_path, start_time, true);
106+
writeFileSync(dir_progress_file, progress_msg);
107+
108+
let finished_msg: string = new Date().toLocaleString() + ": Extracted a total of " + num_extracted.toString() + " .ts files out of " + num_ts_files + " from directory:\n\t\t\t\t\t\t" + dir_path + "\n";
109+
finished_msg += new Date().toLocaleString() + ": Finished processing directory:\n\t\t\t\t\t\t" + this.m_CommandLineValues.Dir + "\n";
110+
appendFileSync(ExtractFeaturesTask.log_file, finished_msg, Common.UTF8);
111+
appendFileSync(this.m_CommandLineValues.logDir + "NumExtracted.log", num_extracted.toString() + "/" + num_processed.toString() + "\n", Common.UTF8)
112+
return num_extracted;
113+
}
114+
115+
/**
116+
* Generates a progress message. Meant to be used by ExtractFeaturesTask.processDir.
117+
* @param num_processed - the number of .ts files processed so far.
118+
* @param num_extracted - the number of .ts files extracted so far.
119+
* @param num_ts_files - the total number of .ts files in the directory.
120+
* @param dir_path - the directories' path.
121+
* @param start_time - the time when the processing started.
122+
* @param finished - a flag that indicates whether the directory has been wholy processed or not.
123+
*/
124+
private static getProgressMessage(num_processed:number, num_extracted:number, num_ts_files:number, dir_path:string, start_time:Date, finished:boolean) {
125+
let current_time: Date = new Date();
126+
let progress = (num_processed/num_ts_files*100) | 0;
127+
let progress_msg: string = "TSExtractor is processing directory " + dir_path + ": " + progress + "%\n";
128+
let processed_msg: string = "Processed " + num_processed.toString() + "/" + num_ts_files.toString() + " .ts files\n";
129+
let extracted_msg: string = "Extracted " + num_extracted.toString() + "/" + num_processed.toString() + " .ts files\n";
130+
let start_time_msg: string = "Processing started:\n\t\t\t\t\t" + start_time.toLocaleString() + "\n";
131+
let current_time_msg: string = "Last updated:\n\t\t\t\t\t" + current_time.toLocaleString() + "\n";
132+
if (finished) {
133+
progress_msg = "TSExtractor finished processing directory " + dir_path + ": " + progress + "%\n";
134+
current_time_msg = "Processing finished:\n\t\t\t\t\t" + current_time.toLocaleString() + "\n";
135+
let time_difference = new splittedTimeDifference(start_time, current_time);
136+
current_time_msg += "Total time:\n\t\t\t\t\t";
137+
if (time_difference.days > 0) current_time_msg += time_difference.days + " days, ";
138+
if (time_difference.hours > 0) current_time_msg += time_difference.hours + " hours, ";
139+
if (time_difference.minutes > 0) current_time_msg += time_difference.minutes + " minutes, ";
140+
current_time_msg = current_time_msg + time_difference.seconds + " seconds.\n";
141+
}
142+
return progress_msg + processed_msg + extracted_msg + "\n" + start_time_msg + current_time_msg;
143+
}
144+
145+
/**
146+
* Renders an array of ProgramFeatures to printable strings.
147+
* @param features - the program features.
148+
*/
149+
private static featuresToString(features: Array<ProgramFeatures>): Array<string> {
150+
var strings = new Array<string>();
151+
for (let singleIdFeatures of features) {
152+
var toPrint = singleIdFeatures.toString();
153+
strings.push(toPrint);
154+
}
155+
return strings;
156+
}
157+
}
158+
159+
/**
160+
* Recursively walks a dir.
161+
* @param dir - the directory.
162+
* @param callback - the function to be applied on each file in the directory and in its subdirectories.
163+
*/
164+
function walkDir(dir, callback) {
165+
fs.readdirSync(dir).forEach( f => {
166+
let dirPath = path.join(dir, f);
167+
let isDirectory = fs.statSync(dirPath).isDirectory();
168+
isDirectory ?
169+
walkDir(dirPath, callback) : callback(path.join(dir, f));
170+
});
171+
};
172+
173+
/**
174+
* Calculates the time difference between two given dates in a human-readable form.
175+
* @param date1 - the first date.
176+
* @param date2 - the second date.
177+
*/
178+
function splittedTimeDifference(date1: Date, date2: Date) {
179+
let time_difference: number = date2.getTime() - date1.getTime();
180+
this.seconds = (time_difference / 1000) | 0;
181+
this.minutes = (time_difference / (1000*60)) | 0;
182+
this.hours = (time_difference / (1000*60*60)) | 0;
183+
this.days = (time_difference / (1000*60*60*24)) | 0;
184+
this.seconds = this.seconds - 60*this.minutes;
185+
this.minutes = this.minutes - 60*this.hours;
186+
this.hours = this.hours - 24*this.days;
187+
}

0 commit comments

Comments
 (0)