[Pypi-checkins] r934 - trunk/appengine

[martin.von.loewis]

Author: martin.von.loewis
Date: Mon Jul 11 23:31:35 2011
New Revision: 934

Modified:
   trunk/appengine/stats.py
Log:
Deal with delayed downloads.


Modified: trunk/appengine/stats.py
==============================================================================
--- trunk/appengine/stats.py	(original)
+++ trunk/appengine/stats.py	Mon Jul 11 23:31:35 2011
@@ -1,5 +1,5 @@
 # stdlib
-import datetime, bz2, csv, re, cStringIO, cPickle
+import datetime, bz2, csv, re, cStringIO, cPickle, logging
 from collections import defaultdict
 # GAE
 from google.appengine.api.labs import taskqueue
@@ -30,6 +30,10 @@
     now = datetime.datetime.utcnow()
     return "%s-%.2d-%.2d" % (now.year, now.month, now.day)
 
+def nextday(day):
+    day = datetime.date(*map(int,day.split('-')))+datetime.timedelta(days=1)
+    return "%s-%.2d-%.2d" % (day.year, day.month, day.day)
+
 def mkbz2(entries):
     downloads = entries.items()
     downloads.sort()
@@ -52,6 +56,17 @@
     old = model.Stats.all().filter('day = ', day).fetch(1)
     if old:
         old = old[0]
+        if old.partial is None:
+            # stats have already been integrated.
+            # assign downloads to the next day
+            todo = model.Download.all().filter('day = ', day).fetch(100)
+            newday = nextday(old.day)
+            logging.warning('%d extra downloads for %s, reassigning to %s' % 
+                            (len(todo), old.day, newday))
+            for d in todo:
+                d.day = newday
+                d.put()
+            return
         if len(old.partial) > 500000:
             # argh. need to make multiple files to fit into Google blob limits
             partno = 1
@@ -71,6 +86,7 @@
         key = download.project,download.name,agent
         entries[key] += 1
         deletable.append(download)
+    logging.info("integrated %d stats for %s" % (len(todo), day))
     if len(todo) == 100:
         # Partial results. Save them
         data = mkbz2(entries)