Merge pull request #4 from linkvt/feature-pipeline-changes

Feature pipeline changes + DEV language extactor

Author: BlackDark

Main.py

@@ -1,14 +1,16 @@
 import argparse
 
-from classifier.FeatureExtractor import FeatureExtractor
+from classifier.DecisionTreeClassifier import DecisionTreeClassifier
 from classifier.GithubAuthentification import GithubAuthentification
 from classifier.InputParser import InputParser
-from classifier.DecisionTreeClassifier import DecisionTreeClassifier
+from classifier.feature_extraction.FeatureExtractionPipeline import FeatureExtractionPipeline
 
 parser = argparse.ArgumentParser(description='Program which analyses github repositories into categories.')
-parser.add_argument('-f', '--file', dest="filepath", help='The file location of the input file', nargs='?', required=True,
+parser.add_argument('-f', '--file', dest="filepath", help='The file location of the input file', nargs='?',
+                    required=True,
                     metavar='FILE')
-parser.add_argument('-t', '--train', dest="train", help='Specifies the program to train with the given data', action="store_true")
+parser.add_argument('-t', '--train', dest="train", help='Specifies the program to train with the given data',
+                    action="store_true")
 
 args = parser.parse_args()
 
@@ -25,11 +27,11 @@
     # build the samples
     for (url, current_label) in zip(splitted_urls, labels):
         current_repo = github_connection.get_repo(url)
-        print('<Testing> Read repo name:{} with label {}'.format( current_repo.name, current_label))
-        features = FeatureExtractor(current_repo).extract_features()
+        print('<Testing> Read repo name:{} with label {}'.format(current_repo.name, current_label))
+        features = FeatureExtractionPipeline(current_repo).extract_features()
         print('Extracted features: ', features)
         samples.append(features)
 
     clf.fit(samples, labels)
-    predicted_label = clf.predict([[5]])
-    print("Predict label for repo with 5 forks: {}".format(predicted_label))
+    predicted_label = clf.predict_with_values([[5, 1]])
+    print('Prediction: ', predicted_label)

classifier/DecisionTreeClassifier.py

@@ -6,7 +6,32 @@ def __init__(self):
         self.clf = tree.DecisionTreeClassifier()
 
     def fit(self, samples, labels):
-        self.clf = self.clf.fit(samples, labels)
+        """
+        samples eq. -> [[Feature1, Feature2], [Feature1, Feature2]]
+        :param samples:
+        :param labels:
+        :return:
+        """
+        self.clf = self.clf.fit(self._map_input(samples), labels)
 
     def predict(self, samples):
+        """
+        The sample input contains only the necessary values for scikit input
+        :param samples:
+        :return:
+        """
+        return self.clf.predict(self._map_input(samples))
+
+    def predict_with_values(self, samples):
         return self.clf.predict(samples)
+
+    def _map_input(self, samples):
+        """
+        We need a specific input for the scikit classifier
+        :param samples:
+        :return:
+        """
+        mapped_samples = []
+        for sample in samples:
+            mapped_samples.append([feature.value for feature in sample])
+        return mapped_samples

classifier/Feature.py

@@ -1,4 +1,10 @@
 class Feature:
     def __init__(self, name, value):
-        self._name = name
-        self._value = value
+        self.name = name
+        self.value = value
+
+    def __str__(self):
+        return 'Feature[' + self.name + ', ' + str(self.value) + ']'
+
+    def __repr__(self):
+        return self.__str__()

classifier/FeatureExtractor.py

@@ -0,0 +1,24 @@
+import abc
+from itertools import chain
+
+from classifier import Feature
+from classifier.feature_extraction.FeatureExtractor import FeatureExtractor
+
+
+class FeatureCategory:
+    """
+    Each category is a representation for a own weightable feature set.
+    This can be for example be used for own trees.
+    """
+
+    def __init__(self, repo):
+        self.repo = repo
+        self.features = list(chain.from_iterable(
+            (extractor.extract_features() for extractor in self._get_feature_extractors())))
+
+    def extract(self) -> [Feature]:
+        return self.features
+
+    @abc.abstractmethod
+    def _get_feature_extractors(self) -> [FeatureExtractor]:
+        raise AssertionError('Must be overridden')

classifier/feature_extraction/FeatureCategory.py

@@ -0,0 +1,22 @@
+from itertools import chain
+
+from github.Repository import Repository
+
+from classifier.Feature import Feature
+from classifier.feature_extraction import FeatureCategory
+from classifier.feature_extraction.MainCategory import MainCategory
+
+CATEGORIES = [
+    MainCategory
+]
+
+
+class FeatureExtractionPipeline:
+    def __init__(self, repo: Repository):
+        self._repo = repo
+
+    def extract_features(self) -> [Feature]:
+        return list(chain.from_iterable((category.extract() for category in self.extract_features_in_categories())))
+
+    def extract_features_in_categories(self) -> [FeatureCategory]:
+        return [category(self._repo) for category in CATEGORIES]

classifier/feature_extraction/FeatureExtractionPipeline.py

@@ -0,0 +1,21 @@
+import abc
+
+from github.Repository import Repository
+
+from classifier import Feature
+
+
+class FeatureExtractor:
+    """
+    Base class for all feature extractors.
+    """
+
+    def __init__(self, repo: Repository):
+        self._repo = repo
+
+    @abc.abstractmethod
+    def extract_features(self) -> [Feature]:
+        """
+        :return: Feature
+        """
+        raise NotImplementedError('Main class should not be called for feature extraction!')

classifier/feature_extraction/FeatureExtractor.py

@@ -0,0 +1,7 @@
+from classifier.Feature import Feature
+from classifier.feature_extraction.FeatureExtractor import FeatureExtractor
+
+
+class ForkExtractor(FeatureExtractor):
+    def extract_features(self) -> [Feature]:
+        return [Feature('Test feature with forks', self._repo.forks)]

classifier/feature_extraction/ForkExtractor.py

@@ -0,0 +1,10 @@
+from classifier.feature_extraction.ForkExtractor import ForkExtractor
+from classifier.feature_extraction.FeatureCategory import FeatureCategory
+from classifier.feature_extraction.FeatureExtractor import FeatureExtractor
+from classifier.feature_extraction.language.AllLanguageFeatureExtractor import AllLanguageFeatureExtractor
+from classifier.feature_extraction.language.LanguageDEVFeatureExtractor import LanguageDEVFeatureExtractor
+
+
+class MainCategory(FeatureCategory):
+    def _get_feature_extractors(self) -> [FeatureExtractor]:
+        return [ForkExtractor(self.repo), LanguageDEVFeatureExtractor(self.repo), AllLanguageFeatureExtractor(self.repo)]

classifier/feature_extraction/MainCategory.py

@@ -0,0 +1,40 @@
+import yaml
+from github.Repository import Repository
+
+from classifier.Feature import Feature
+from classifier.feature_extraction.FeatureExtractor import FeatureExtractor
+
+
+class AllLanguageFeatureExtractor(FeatureExtractor):
+    """
+    The languages returned from github are mapped to the byte size of usage.
+    Eq {'Python': 98564, 'R': 4914}
+    Currently there are 223 languages
+    """
+    LANGUAGES = []
+
+    def __init__(self, repo: Repository):
+        super().__init__(repo)
+        self._init_languages()
+        self._languageToProbability = {el: 0 for el in AllLanguageFeatureExtractor.LANGUAGES}
+
+    def _init_languages(self):
+        if not AllLanguageFeatureExtractor.LANGUAGES:
+            with open('languages.yml', 'r') as f:
+                doc = yaml.load(f)
+                AllLanguageFeatureExtractor.LANGUAGES = [language for language in doc]
+
+    def dict(self):
+        return self._languageToProbability
+
+    def extract_features(self) -> [Feature]:
+        languages = self._repo.get_languages()
+        total_size = sum(languages.values())
+
+        for language in languages:
+            if language in self._languageToProbability:
+                self._languageToProbability[language] = languages[language] / total_size
+            else:
+                print('Language "' + language + '" is not registered in the algorithm.')
+
+        return [Feature('Language: ' + key, value) for key, value in self._languageToProbability.items()]