stats gathering: added counters to upload,download,mutablewatcher

counting number of operations, and for immutable files, bytes transferred

src/allmydata/client.py

@@ -159,10 +159,10 @@ class Client(node.Node, testutil.PollMixin):
         helper_furl = self.get_config("helper.furl")
         convergence_s = self.get_or_create_private_config('convergence', _make_secret)
         self.convergence = base32.a2b(convergence_s)
-        self.add_service(Uploader(helper_furl))
-        self.add_service(Downloader())
+        self.add_service(Uploader(helper_furl, self.stats_provider))
+        self.add_service(Downloader(self.stats_provider))
         self.add_service(Checker())
-        self.add_service(MutableWatcher())
+        self.add_service(MutableWatcher(self.stats_provider))
         def _publish(res):
             # we publish an empty object so that the introducer can count how
             # many clients are connected and see what versions they're

src/allmydata/download.py

@@ -1038,9 +1038,10 @@ class Downloader(service.MultiService):
     name = "downloader"
     MAX_DOWNLOAD_STATUSES = 10
 
-    def __init__(self):
+    def __init__(self, stats_provider=None):
         service.MultiService.__init__(self)
         self._all_downloads = weakref.WeakKeyDictionary()
+        self.stats_provider = stats_provider
         self._recent_download_status = []
 
     def download(self, u, t):
@@ -1050,6 +1051,11 @@ class Downloader(service.MultiService):
         t = IDownloadTarget(t)
         assert t.write
         assert t.close
+
+        if self.stats_provider:
+            self.stats_provider.count('downloader.files_downloaded', 1)
+            self.stats_provider.count('downloader.bytes_downloaded', u.get_size())
+
         if isinstance(u, uri.LiteralFileURI):
             dl = LiteralDownloader(self.parent, u, t)
         elif isinstance(u, uri.CHKFileURI):

src/allmydata/mutable.py

@@ -1830,8 +1830,9 @@ class MutableWatcher(service.MultiService):
     MAX_RETRIEVE_STATUSES = 20
     name = "mutable-watcher"
 
-    def __init__(self):
+    def __init__(self, stats_provider=None):
         service.MultiService.__init__(self)
+        self.stats_provider = stats_provider
         self._all_publish = weakref.WeakKeyDictionary()
         self._recent_publish_status = []
         self._all_retrieve = weakref.WeakKeyDictionary()
@@ -1840,6 +1841,9 @@ class MutableWatcher(service.MultiService):
     def notify_publish(self, p):
         self._all_publish[p] = None
         self._recent_publish_status.append(p.get_status())
+        if self.stats_provider:
+            self.stats_provider.count('mutable.files_published', 1)
+            #self.stats_provider.count('mutable.bytes_published', p._node.get_size())
         while len(self._recent_publish_status) > self.MAX_PUBLISH_STATUSES:
             self._recent_publish_status.pop(0)
 
@@ -1855,6 +1859,9 @@ class MutableWatcher(service.MultiService):
     def notify_retrieve(self, r):
         self._all_retrieve[r] = None
         self._recent_retrieve_status.append(r.get_status())
+        if self.stats_provider:
+            self.stats_provider.count('mutable.files_retrieved', 1)
+            #self.stats_provider.count('mutable.bytes_retrieved', r._node.get_size())
         while len(self._recent_retrieve_status) > self.MAX_RETRIEVE_STATUSES:
             self._recent_retrieve_status.pop(0)
 

src/allmydata/upload.py

@@ -1194,8 +1194,9 @@ class Uploader(service.MultiService):
     URI_LIT_SIZE_THRESHOLD = 55
     MAX_UPLOAD_STATUSES = 10
 
-    def __init__(self, helper_furl=None):
+    def __init__(self, helper_furl=None, stats_provider=None):
         self._helper_furl = helper_furl
+        self.stats_provider = stats_provider
         self._helper = None
         self._all_uploads = weakref.WeakKeyDictionary()
         self._recent_upload_status = []
@@ -1229,6 +1230,11 @@ class Uploader(service.MultiService):
             precondition(isinstance(default_params, dict), default_params)
             precondition("max_segment_size" in default_params, default_params)
             uploadable.set_default_encoding_parameters(default_params)
+
+            if self.stats_provider:
+                self.stats_provider.count('uploader.files_uploaded', 1)
+                self.stats_provider.count('uploader.bytes_uploaded', size)
+
             if size <= self.URI_LIT_SIZE_THRESHOLD:
                 uploader = LiteralUploader(self.parent)
             elif self._helper: