[pypy-commit] extradoc extradoc: in-progress

[arigo]

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r4956:e29ffc17e9b2
Date: 2013-03-15 09:19 -0700
http://bitbucket.org/pypy/extradoc/changeset/e29ffc17e9b2/

Log:	in-progress

diff --git a/talk/pycon2013/pypy_without_gil/message_passing.py b/talk/pycon2013/pypy_without_gil/message_passing.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon2013/pypy_without_gil/message_passing.py
@@ -0,0 +1,28 @@
+import thread, Queue
+
+
+def compute(queue):
+    #
+    for j in range(10):
+        for k in range(10**6):
+            pass
+        queue.put(1)
+    #
+    queue.put("done")
+
+
+queue = Queue.Queue()
+for i in range(10):
+    thread.start_new_thread(compute, (queue,))
+
+running_threads = 10
+num = 0
+
+while running_threads > 0:
+    item = queue.get()
+    if item == "done":
+        running_threads -= 1
+    else:
+        num += 1
+
+print num
diff --git a/talk/pycon2013/pypy_without_gil/multithreading.py b/talk/pycon2013/pypy_without_gil/multithreading.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon2013/pypy_without_gil/multithreading.py
@@ -0,0 +1,21 @@
+import thread
+
+
+def compute(lock):
+    #
+    for k in xrange(10**7):
+        pass
+    #
+    lock.release()
+
+
+all_locks = [thread.allocate_lock() for i in range(10)]
+
+for i in range(10):
+    lock = all_locks[i]
+    lock.acquire()
+    thread.start_new_thread(compute, (lock,))
+
+for i in range(10):
+    lock = all_locks[i]
+    lock.acquire()
diff --git a/talk/pycon2013/pypy_without_gil/multithreading2.py b/talk/pycon2013/pypy_without_gil/multithreading2.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon2013/pypy_without_gil/multithreading2.py
@@ -0,0 +1,27 @@
+import thread
+
+
+def compute(lock):
+    global num
+    #
+    for j in range(10):
+        for k in range(10**6):
+            pass
+        num += 1   # not actually safe!  needs a lock
+    #
+    lock.release()
+
+
+num = 0
+all_locks = [thread.allocate_lock() for i in range(10)]
+
+for i in range(10):
+    lock = all_locks[i]
+    lock.acquire()
+    thread.start_new_thread(compute, (lock,))
+
+for i in range(10):
+    lock = all_locks[i]
+    lock.acquire()
+
+print num
diff --git a/talk/pycon2013/pypy_without_gil.txt b/talk/pycon2013/pypy_without_gil/pypy_without_gil.txt
rename from talk/pycon2013/pypy_without_gil.txt
rename to talk/pycon2013/pypy_without_gil/pypy_without_gil.txt
--- a/talk/pycon2013/pypy_without_gil.txt
+++ b/talk/pycon2013/pypy_without_gil/pypy_without_gil.txt
@@ -9,38 +9,101 @@
 
 - thanks to contributors for STM
 
+
 Problem
 -------
 
-* a complex, large program that does stuff
+* An existing complex, large program that does stuff
 
-* stuff is mostly independent
+* "stuff" consists of bits that are mostly independent from each other
 
 |pause|
 
 * ... but not quite
 
-* shared mutable state, however, not too much
 
-* we want to parallelize the problem
+Problem
+-------
+
+* We want to parallelize the program to use all these cores
+
+* We have some shared mutable state
+
+|pause|
+
+* Not too much of it --- otherwise, no chance for parallelism
+
+* But still some
+
 
 Classic solutions
 -----------------
 
-* multithreading
+Bare-metal multi-threading:
 
-  * large shared state (tons of lock)
+* large shared state
 
-  * no shared mutable state (copying, keeping in sync)
+* needs careful usage of locks
 
-* MPI - message passing, with limited shared state
+* mostly hindered by the GIL in CPython
+  (but not in Jython)
+
+
+Classic solutions
+-----------------
+
+Multi-processing:
+
+* no shared mutable state at all
+
+* copying, keeping in sync are your problem
+
+* memory usage is often multiplied (unless you're lucky with ``fork``)
+
+
+Classic solutions
+-----------------
+
+A range of intermediate solutions:
+
+* MPI: message passing, with limited shared state
+
+* etc.: tons of experiments that never caught on
+
+
+Classic solutions
+-----------------
+
+The typical solution for web servers:
+
+* run independent processes
+
+* share data only via the database
+
+* the database itself handles concurrency with transactions
+
 
 Demo
 ---------------------------
 
-- demo: ~/br/stm-thread-2/64compiled/pypy-c-r61735+-stm
+* ``pypy-stm``
 
-  * run some random multithreaded program
+* internally based on "transactions" (STM, HTM)
+
+
+Demo
+---------------------------
+
+- demo: 
+
+  * multithreading.py
+
+  * multithreading2.py
+
+  * message_passing.py
+
+  * transactions2.py
+
 
 
 - missing in pypy-stm:
@@ -102,6 +165,9 @@
 ---------------------------
 
 
+- xxx memory usage good
+
+
 - based on (and fully compatible with) threads
 
   * existing multithreaded programs work
@@ -129,7 +195,7 @@
 
   * The idea is that we fix only XX% of the bugs and we are done
 
-  * Maybe some new "performance debugger" tools might help us too
+  * Maybe some new "performance debugger" tools might help too
 
 
 - TigerQuoll, 1st March: same idea with JavaScript (for servers)
@@ -154,11 +220,4 @@
   * the events are run with ``atomic``, so that they appear to run
     serially
 
-  * this tweaked event loop should work with just any existing program
-
-
-- Futures
-
-  * e.g. twisted's deferred
-
-  ...
+  * does not rely on any change to the user program
diff --git a/talk/pycon2013/pypy_without_gil/transactions0.py b/talk/pycon2013/pypy_without_gil/transactions0.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon2013/pypy_without_gil/transactions0.py
@@ -0,0 +1,13 @@
+import transaction
+
+
+
+def compute():
+    for k in range(10**7):
+        pass
+
+
+for i in range(10):
+    transaction.add(compute)
+
+transaction.run()
diff --git a/talk/pycon2013/pypy_without_gil/transactions1.py b/talk/pycon2013/pypy_without_gil/transactions1.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon2013/pypy_without_gil/transactions1.py
@@ -0,0 +1,24 @@
+import transaction
+
+
+
+class MyStuff(object):
+    counter = 0
+
+def increment_class_counter():
+    MyStuff.counter += 1
+
+
+def compute():
+    for j in range(10):
+        for k in range(10**6):
+            pass
+        transaction.add(increment_class_counter)
+
+
+for i in range(10):
+    transaction.add(compute)
+
+transaction.run()
+
+print MyStuff.counter
diff --git a/talk/pycon2013/pypy_without_gil/transactions2.py b/talk/pycon2013/pypy_without_gil/transactions2.py
new file mode 100644
--- /dev/null
+++ b/talk/pycon2013/pypy_without_gil/transactions2.py
@@ -0,0 +1,18 @@
+import transaction
+
+
+def do_stuff(x):
+    # huge piece of code doing tons of messy things
+    for j in range(10**6):
+        assert x != 42
+
+
+def do_stuff_for_all(lst):
+    for x in lst:
+        do_stuff(x)
+    #for x in lst:
+    #    transaction.add(do_stuff, x)
+    #transaction.run()
+
+
+do_stuff_for_all(range(20))