[python-win32] multiprocessing pickle problem in win32

[Rob Brown-Bayliss]

Hi

I was playing with multiple mysql connection in separate processes
with the multiprocessing module.

When I run the following code in linux all goes well.  But when I try
in win32 I get a pickle error saying it cant pickle None Type when I
start the process.

Python 2.6 on both platforms.

Thanks in advance.

from __future__ import print_function
import sys
import os
import platform
import MySQLdb
import multiprocessing
import time
import cPickle as Pickle
VERBOSE = False

class SQL_Query(object):
	def __init__(self, SQL, callback):
		self.SQL = SQL
		self.callback = callback

class Connection(multiprocessing.Process):

	def __init__(self,in_queue, out_queue):
		multiprocessing.Process.__init__(self)
		self.in_queue = in_queue
		self.out_queue = out_queue
		self.counter =  10 #int(self.conf["db_num_queries"]) # Number of
queries to handle before closing down.
		self.all_ok = True
		self.con = self.get_connection()
	
	def escape(self, string):
		return MySQLdb.escape(string)

	def execute(self, sql):
		"Return results of sql from database, sql must be valid SQL as no
checking is done."
		if VERBOSE:
			print (sql)
		res = True
		last_id = 0
		self.counter -= 1
		if self.con:
			try:
				cur = self.con.cursor()
				cur.execute("START TRANSACTION")
				cur.execute(sql)
				res = cur.fetchall()
				self.con.commit()
				last_id = cur.execute("SELECT LAST_INSERT_ID()")
				cur.close()
			except Exception as e :
				res = False  # For tracking insert failure etc...
				self.all_ok = False
				self.con.rollback()
			self.out_queue.put((res, last_id))
		else:
			util.error_ui(message = "A Connection has not been established with
the database.")

	def get_connection(self):
		self.con = ""
		hst = "127.0.0.1"
		usr = "usr"
		pwd = "pass"
		database = "db"
		try:
			self.con = MySQLdb.connect(host=hst, user=usr, passwd=pwd, db=database)
		except Exception as e:
			pass
		return self.con
	
	def run(self):
		proc_name = self.name
		while self.counter > 0:
			next_sql = self.in_queue.get()
			print(self, self.counter, next_sql.SQL)
			if next_sql is None:
				# Stop Process
				return
			result = self.execute(next_sql.SQL)
			self.out_queue.put(result)
		self.close_down()
		self.out_queue.put(("RESTART", self.name))
		return
	
#
# clean up
#

	def close_down(self, arg1 = None):
		if self.con:
			self.con.close()


if __name__ == "__main__":
	in_queue = multiprocessing.Queue()
	out_queue = multiprocessing.Queue()
	pool = []
	pool.append(Connection(in_queue, out_queue))
	pool.append(Connection(in_queue, out_queue))
	pool.append(Connection(in_queue, out_queue))


	print(pool)
	for c in pool:
		c.start()

	for i in range(100):
		print(i)
		s = SQL_Query("SELECT * FROM customer ", "-")
		#time.sleep(0.015)
		in_queue.put(s)
	
	while True:
		o = out_queue.get()
		if o:
			if o[0]=="RESTART":
				print(o)
				print(pool)
				for c in pool:
					if c.name == o[1]:
						print(c)
						c.join()
						c.terminate()
						pool.remove(c)
						nc = Connection(in_queue, out_queue)
						nc.start()
						pool.append(nc)



-- 
--

Rob