Changeset 480:c2a9b31e1fd5

Timestamp:

09/02/08 20:48:55 (4 months ago)

Author:

Brian Warner <warner@allmydata.com>

branch:

default

Message:

gatterer: reclassify everything that isn't already classified. Closes #94.

Files:

• ChangeLog (modified) (1 diff)
• foolscap/logging/gatherer.py (modified) (4 diffs)
• foolscap/pb.py (modified) (1 diff)
• foolscap/test/test_logging.py (modified) (5 diffs)

ChangeLog

@@ -1 +1 @@
-2008-09-02  Brian Warner  <warner@lothar.com>
+
+        * foolscap/pb.py (Tub._log_gatherer_connected): use callRemoteOnly
+        to pass the logport to the gatherer: we don't need to hear about
+        any problems it has.
+
+        * foolscap/logging/gatherer.py
+        (IncidentGathererService.classify_stored_incidents): reclassify
+        everything that isn't already present in one of the classified/*
+        files. This makes it a lot easier to iterate over the [start
+        gatherer; see what is unknown; update classifiers; remove
+        classified/unknown; repeat] loop. Also log classification events
+        better. Closes #94.
+        * foolscap/test/test_logging.py (IncidentGatherer.test_emit): test it
-
-        * foolscap/test/common.py (PollMixin): replace the use of chained

foolscap/logging/gatherer.py

@@ -436 +436 @@
-        if not os.path.isdir(outputdir):
-            os.makedirs(outputdir)
-    
+
-        GatheringBase.startService(self)
-
-    def classify_stored_incidents(self, indir):
-        stdout = self.stdout or sys.stdout
-
-
+
+
+
+
+
+
+
+
+
+
+
-        for tubid_s in os.listdir(indir):
-            nodedir = os.path.join(indir, tubid_s)
@@ -448 +457 @@
-                if fn.startswith("incident-"):
-                    abs_fn = os.path.join(nodedir, fn)
+                    if abs_fn in already:
+                        continue
-                    incident = self.load_incident(abs_fn)
-                    rel_fn = os.path.join("incidents", tubid_s, fn)
-                    self.classify_incident(rel_fn, tubid_s, incident)
+                    count += 1
+        print >>stdout, "done classifying %d stored incidents" % count
-
-    def load_incident(self, abs_fn):
@@ -480 +493 @@
-    def new_incident(self, abs_fn, rel_fn, tubid_s, incident):
-        stdout = self.stdout or sys.stdout
-
-
+
-        self.incidents_received += 1
-
-    def classify_incident(self, rel_fn, tubid_s, incident):
+        stdout = self.stdout or sys.stdout
-        categories = set()
-        for f in self.classifiers:
@@ -499 +512 @@
-            f.write(rel_fn + "\n")
-            f.close()
+        print >>stdout, "classified %s as [%s]" % (rel_fn, ",".join(categories))
-        return categories
-

foolscap/pb.py

@@ -391 +391 @@
-            # RILogGatherer.logport requires a string for nodeid=
-            tubID = '<unauth>'
-
+Only
-
-

foolscap/test/test_logging.py

@@ -1010 +1010 @@
-        # at startup.
-
-        # give the call to remote_logport a chance to retire
-        d.addCallback(self.stall, 0.5)
-        d.addCallback(lambda res: ig.disownServiceParent())
+
+        def classify_boom(nodeid_s, (header,events)):
+            if "boom" in header["trigger"].get("message",""):
+                return "boom"
+        def classify_foom(nodeid_s, (header,events)):
+            if "foom" in header["trigger"].get("message",""):
+                return "foom"
-
-        incident_d2 = defer.Deferred()
-        def _update_classifiers(res):
-            self.remove_classified_incidents(ig)
-            def classify_boom(nodeid_s, (header,events)):
-                if "boom" in header["trigger"].get("message",""):
-                    return "boom"
-            def classify_foom(nodeid_s, (header,events)):
-                if "foom" in header["trigger"].get("message",""):
-                    return "foom"
-            ig2 = self.create_incident_gatherer(basedir, [classify_boom])
-            ig2.add_classifier(classify_foom)
@@ -1054 +1053 @@
-            self.failIf(os.path.exists(unknowns_fn))
-        d.addCallback(_new_incident2)
+        d.addCallback(lambda res: self.ig2.disownServiceParent())
+
+        # if we remove just classified/boom, then those incidents should be
+        # reclassified
+
+        def _remove_boom_incidents(res):
+            booms_fn = os.path.join(ig.basedir, "classified", "boom")
+            os.remove(booms_fn)
+
+            ig2a = self.create_incident_gatherer(basedir, [classify_boom,
+                                                           classify_foom])
+            ig2a.setServiceParent(self.parent)
+            self.ig2a = ig2a
+
+            # now classified/boom should be back, and the other files should
+            # have been left untouched
+            booms = [fn.strip() for fn in open(booms_fn,"r").readlines()]
+            self.failUnlessEqual(len(booms), 1)
+        d.addCallback(_remove_boom_incidents)
+        d.addCallback(lambda res: self.ig2a.disownServiceParent())
-
-        # and if we remove the classification functions (but do *not* remove
@@ -1059 +1078 @@
-        # anything
-
-        d.addCallback(self.stall, 0.5)
-        d.addCallback(lambda res: self.ig2.disownServiceParent())
-        def _update_classifiers_again(res):
-            ig3 = self.create_incident_gatherer(basedir)
@@ -1077 +1094 @@
-        d.addCallback(_update_classifiers_again)
-
-        # give the call to remote_logport a chance to retire
-        d.addCallback(self.stall, 0.5)
-        d.addCallback(lambda res: self.ig3.disownServiceParent())
-
@@ -1099 +1114 @@
-        d.addCallback(lambda res:
-                      self.poll(lambda : self.ig4.incidents_received == 2))
-
-        d.addBoth(self.stall, 0.5) # allow shutdown even on error
-
-        return d