Merge pull request mozilla-services#3077 from twobraids/common-corr

Fixes Bug 1209526 - the correlation scripts as TransformRules + correlations_app

scripts/integration-test.sh

@@ -126,6 +126,7 @@ function cleanup() {
   rm -rf ./primaryCrashStore/ ./processedCrashStore/
   rm -rf ./crashes/
   rm -rf ./submissions
+  rm -rf ./correlations
 
   echo "INFO: Terminating background jobs"
   echo "  any kill usage errors below may be ignored"
@@ -527,5 +528,48 @@ do
 
 done
 
+# start correlations integration tests
+echo "INFO: starting up *** correlations test *** "
+echo "      * in this test, we ensure the correlations reports app produces the correct output in json form"
+
+mkdir ./correlations
+
+socorro correlations \
+--source.crashstorage_class=socorro.collector.submitter_app.SubmitterFileSystemWalkerSource \
+--source.search_root=testcrash/processed \
+--new_crash_source.new_crash_source_class=""  \
+--global.correlations.path=./correlations \
+--global.correlations.core.output_class=socorro.analysis.correlations.core_count_rule.JsonFileOutputForCoreCounts \
+--global.correlations.interesting.output_class=socorro.analysis.correlations.interesting_rule.JsonFileOutputForInterestingModules  \
+--producer_consumer.number_of_threads=10 \
+--destination.crashstorage_class=socorro.external.crashstorage_base.NullCrashStorage \
+--global.correlation.min_count_for_inclusion=1 \
+> correlations.log 2>&1
+
+if [ $? = 1 ]
+then
+    echo "ERROR: correlations logged errors"
+    echo "***** BEGIN correlation log *****"
+    cat correlations.log
+    echo "***** END correlation log *****"
+    fatal 1 "ERROR: correlations produced unexpected output"
+fi
+
+check_for_logged_fatal_errors $? correlations
+
+diff ./correlations/20151130 ./testcrash/correlations-integration-correct-output >correlation.diff
+if [ $? = 1 ]
+then
+    # something went wrong
+    echo "ERROR: correlations produced unexpected output"
+    echo "***** BEGIN correlation log *****"
+    cat correlations.log
+    echo "***** BEGIN correlation diff *****"
+    cat correlation.diff
+    echo "***** END correlation diff *****"
+    fatal 1 "ERROR: correlations produced unexpected output"
+fi
+echo "*** correlations *** PASSES INTEGRATION TESTS"
+
 echo "If you are reading this, then ALL the integration tests passed!"
 exit 0

socorro/analysis/correlations/addonids.py

@@ -0,0 +1,106 @@
+#!/usr/bin/python
+# vim: set shiftwidth=4 tabstop=4 autoindent expandtab:
+# ***** BEGIN LICENSE BLOCK *****
+# Version: MPL 1.1/GPL 2.0/LGPL 2.1
+#
+# The contents of this file are subject to the Mozilla Public License Version
+# 1.1 (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.mozilla.org/MPL/
+#
+# Software distributed under the License is distributed on an "AS IS" basis,
+# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+# for the specific language governing rights and limitations under the
+# License.
+#
+# The Original Code is AddonIds.py.
+#
+# The Initial Developer of the Original Code is the Mozilla Foundation.
+# Portions created by the Initial Developer are Copyright (C) 2009
+# the Initial Developer. All Rights Reserved.
+#
+# Contributor(s):
+#   L. David Baron <dbaron@dbaron.org>, Mozilla Corporation (original author)
+#
+# Alternatively, the contents of this file may be used under the terms of
+# either the GNU General Public License Version 2 or later (the "GPL"), or
+# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+# in which case the provisions of the GPL or the LGPL are applicable instead
+# of those above. If you wish to allow use of your version of this file only
+# under the terms of either the GPL or the LGPL, and not to allow others to
+# use your version of this file under the terms of the MPL, indicate your
+# decision by deleting the provisions above and replace them with the notice
+# and other provisions required by the GPL or the LGPL. If you do not delete
+# the provisions above, a recipient may use your version of this file under
+# the terms of any one of the MPL, the GPL or the LGPL.
+#
+# ***** END LICENSE BLOCK *****
+
+import jsondb
+import sys
+import re
+
+__all__ = [
+    "info_for_id"
+]
+
+class AddonInfoMap(jsondb.JsonDB):
+    def put(self, id, name, url):
+        self._map[id] = { "name": name, "url": url }
+    def get(self, id):
+        map = self._map
+        if not id in map:
+            return None
+        entry = map[id]
+        return AddonInfo(name=entry["name"], url=entry["url"])
+
+class AMOAddonInfoMap(AddonInfoMap):
+    def get(self, id):
+        if not id in self._map:
+            self.check_AMO()
+        return AddonInfoMap.get(self, id)
+    def check_AMO(self):
+        # store result whether it's "None" or not
+        # FIXME: WRITE ME
+        pass
+
+class AddonInfo:
+    def __init__(self, name, url):
+        self.name = name
+        self.url = url
+
+# FIXME: There's lots of about:support output in support.mozilla.com
+# forums that has additional extension IDs paired with names.  It would
+# be a good additional source of data.
+local_db = AddonInfoMap("addonids-local")
+amo_db = AMOAddonInfoMap("addonids-amo")
+
+def info_for_id(id):
+    return local_db.get(id) or amo_db.get(id)
+
+# When executed directly...
+if __name__ == '__main__':
+    if len(sys.argv) == 1:
+        # prompt for an addition to the local database.
+        sys.stdout.write("Addon ID: ")
+        id = sys.stdin.readline().rstrip("\n")
+        sys.stdout.write("Name: ")
+        name = sys.stdin.readline().rstrip("\n")
+        sys.stdout.write("Homepage: ")
+        url = sys.stdin.readline().rstrip("\n")
+        local_db.put(id, name, url)
+        local_db.write()
+    elif len(sys.argv) == 2 and sys.argv[1] == "-i":
+        # or, with -i, import the AMO one
+        io = open("amo-ids.txt", "r")
+        for line in io:
+            # Sometimes the last field is \N instead of a string
+            line = re.sub(r'\\N$', r'""', line)
+            [id, number, name] = json.loads("[" + line + "]")
+            if len(id) > 0:
+                url = "https://addons.mozilla.org/addon/" + str(number)
+                amo_db.put(id, name, url)
+        io.close()
+        amo_db.write()
+    else:
+        raise StandardError("unexpected arguments")