Changeset 495:2fa51824742b

Timestamp:

10/14/08 16:14:28 (3 months ago)

Author:

Brian Warner <warner@allmydata.com>

branch:

default

Message:

flogtool classify-incident: new subcommand. Closes #102.

Files:

• ChangeLog (modified) (1 diff)
• foolscap/logging/cli.py (modified) (3 diffs)
• foolscap/logging/gatherer.py (modified) (6 diffs)
• foolscap/logging/incident.py (modified) (2 diffs)
• foolscap/test/test_logging.py (modified) (1 diff)

ChangeLog

@@ -1 +1 @@
-2008-10-14  Brian Warner  <warner@allmydata.com>
+
+        * foolscap/logging/cli.py: new "flogtool classify-incident"
+        subcommand: given an incident, say what categories it falls into.
+        Closes #102.
+        * foolscap/logging/gatherer.py (IncidentGathererService): factor
+        out the classification pieces into IncidentClassifierBase
+        * foolscap/logging/incident.py (IncidentClassifierBase): same
+        (IncidentClassifier.run): support for the new CLI command
+        * foolscap/test/test_logging.py (Incidents.test_classify): test it
-
-        * foolscap/logging/gatherer.py

foolscap/logging/cli.py

@@ -5 +5 @@
-
-import foolscap
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-
-class Options(usage.Options):
@@ -27 +29 @@
-        ("web-viewer", None, WebViewerOptions,
-         "view the logs through a web page"),
+        ("classify-incident", None, ClassifyOptions,
+         "classify a stored Incident file"),
-        ]
-
@@ -46 +50 @@
-def dispatch(command, options):
-    if command == "tail":
-        from foolscap.logging.tail import LogTail
-        lt = LogTail(options)
-        lt.run(options.target_furl)
-
-    elif command == "create-gatherer":
-        from foolscap.logging.gatherer import create_log_gatherer
-        create_log_gatherer(options)
-
-    elif command == "create-incident-gatherer":
-        from foolscap.logging.gatherer import create_incident_gatherer
-        create_incident_gatherer(options)
-
-    elif command == "dump":
-        from foolscap.logging.dumper import LogDumper
-        ld = LogDumper()
-        ld.run(options)
-
-    elif command == "filter":
-        from foolscap.logging.filter import Filter
-        f = Filter()
-        f.run(options)
-
-    elif command == "web-viewer":
-        from foolscap.logging.web import WebViewer
-        wv = WebViewer()
-        wv.run(options)
+
+    elif command == "classify-incident":
+        ic = IncidentClassifier()
+        ic.run(options)
-
-    else:

foolscap/logging/gatherer.py

@@ -12 +12 @@
-import foolscap
-from foolscap.logging.interfaces import RILogGatherer, RILogObserver
+from foolscap.logging.incident import IncidentClassifierBase
-from foolscap.util import get_local_ip_for
-
@@ -391 +392 @@
-        return None
-
-
+, IncidentClassifierBase
-    # create this with 'flogtool create-incident-gatherer BASEDIR'
-    # run this as 'cd BASEDIR && twistd -y gatherer.tac'
@@ -420 +421 @@
-    def __init__(self, classifiers=[], basedir=None, stdout=None):
-        GatheringBase.__init__(self, basedir)
-self.classifiers = []
+IncidentClassifierBase.__init__(self)
-        self.classifiers.extend(classifiers)
-        self.stdout = stdout
-        self.incidents_received = 0 # for tests
-
-    def add_classifier(self, f):
-        # there are old .tac files that call this explicitly
-        self.classifiers.append(f)
-
-
@@ -437 +434 @@
-        if not os.path.isdir(outputdir):
-            os.makedirs(outputdir)
-
+self.basedir
-        self.classify_stored_incidents(indir)
-        GatheringBase.startService(self)
-
-    def add_classify_files(self):
-        for fn in os.listdir(self.basedir):
-            if not (fn.startswith("classify_") and fn.endswith(".py")):
-                continue
-            f = open(os.path.join(self.basedir, fn), "r")
-            localdict = {}
-            exec f in localdict
-            self.add_classifier(localdict["classify_incident"])
-
-    def classify_stored_incidents(self, indir):
@@ -472 +460 @@
-                    incident = self.load_incident(abs_fn)
-                    rel_fn = os.path.join("incidents", tubid_s, fn)
-classify
+move
-                    count += 1
-        print >>stdout, "done classifying %d stored incidents" % count
-
-    def load_incident(self, abs_fn):
-        assert abs_fn.endswith(".bz2")
-        f = bz2.BZ2File(abs_fn, "r")
-        header = pickle.load(f)["header"]
-        events = []
-        while True:
-            try:
-                wrapped = pickle.load(f)
-            except (EOFError, ValueError):
-                break
-            events.append(wrapped["d"])
-        f.close()
-        return (header, events)
-
-    def remote_logport(self, nodeid, publisher):
@@ -504 +478 @@
-    def new_incident(self, abs_fn, rel_fn, tubid_s, incident):
-        stdout = self.stdout or sys.stdout
-classify
+move
-        self.incidents_received += 1
-
-classify
+move
-        stdout = self.stdout or sys.stdout
-
-
-
-
-
-
-
-
-
-
-
+
-        for c in categories:
-            fn = os.path.join(self.basedir, "classified", c)

foolscap/logging/incident.py

@@ -1 +1 @@
-
-
+sys, 
-from zope.interface import implements
+from twisted.python import usage
-from twisted.internet import reactor
-from foolscap.logging.interfaces import IIncidentReporter
@@ -165 +166 @@
-class NonTrailingIncidentReporter(IncidentReporter):
-    TRAILING_DELAY = None
+
+
+class ClassifyOptions(usage.Options):
+    stdout = sys.stdout
+    stderr = sys.stderr
+    synopsis = "Usage: flogtool classify-incident [options] INCIDENTFILE.."
+
+    optParameters = [
+        ("classifier-directory", "c", ".",
+         "directory with classify_*.py functions to import"),
+        ]
+
+    def parseArgs(self, *files):
+        self.files = files
+
+
+class IncidentClassifierBase:
+
+    def __init__(self):
+        self.classifiers = []
+
+    def add_classifier(self, f):
+        # there are old .tac files that call this explicitly
+        self.classifiers.append(f)
+
+    def add_classify_files(self, plugindir):
+        plugindir = os.path.expanduser(plugindir)
+        for fn in os.listdir(plugindir):
+            if not (fn.startswith("classify_") and fn.endswith(".py")):
+                continue
+            f = open(os.path.join(plugindir, fn), "r")
+            localdict = {}
+            exec f in localdict
+            self.add_classifier(localdict["classify_incident"])
+
+    def load_incident(self, abs_fn):
+        assert abs_fn.endswith(".bz2")
+        f = bz2.BZ2File(abs_fn, "r")
+        header = pickle.load(f)["header"]
+        events = []
+        while True:
+            try:
+                wrapped = pickle.load(f)
+            except (EOFError, ValueError):
+                break
+            events.append(wrapped["d"])
+        f.close()
+        return (header, events)
+
+    def classify_incident(self, incident):
+        categories = set()
+        for f in self.classifiers:
+            (header, events) = incident
+            trigger = header["trigger"]
+            c = f(trigger)
+            if c: # allow the classifier to return None, or [], or ["foo"]
+                if isinstance(c, str):
+                    c = [c] # or just "foo"
+                categories.update(c)
+        if not categories:
+            categories.add("unknown")
+        return categories
+
+class IncidentClassifier(IncidentClassifierBase):
+    def run(self, options):
+        self.add_classify_files(options["classifier-directory"])
+        out = options.stdout
+        for f in options.files:
+            abs_fn = os.path.expanduser(f)
+            incident = self.load_incident(abs_fn)
+            categories = self.classify_incident(incident)
+            print >>out, "%s: %s" % (f, ",".join(sorted(categories)))
+

foolscap/test/test_logging.py

@@ -352 +352 @@
-        return d
-
+    def test_classify(self):
+        l = log.FoolscapLogger()
+        l.setIncidentReporterFactory(incident.NonTrailingIncidentReporter)
+        l.setLogDir("logging/Incidents/classify")
+        got_logdir = l.logdir
+        l.msg("foom", level=log.BAD)
+        d = fireEventually()
+        def _check(res):
+            files = [fn for fn in os.listdir(got_logdir) if fn.endswith(".bz2")]
+            self.failUnlessEqual(len(files), 1)
+
+            ic = incident.IncidentClassifier()
+            def classify_foom(trigger):
+                if "foom" in trigger.get("message",""):
+                    return "foom"
+            ic.add_classifier(classify_foom)
+            options = incident.ClassifyOptions()
+            options.parseOptions([os.path.join(got_logdir, fn) for fn in files])
+            options.stdout = StringIO()
+            ic.run(options)
+            out = options.stdout.getvalue()
+            self.failUnless(out.strip().endswith(": foom"), out)
+        d.addCallback(_check)
+        return d
-
-class Observer(Referenceable):