newbie subprocess.Popen performance issues/questions

[Chris Seberino]

On Jun 15, 2:44 am, News123 <news1... at free.fr> wrote:
> ChrisSeberinowrote:
> > I tried to use subprocess.Popen to make my web app do a bunch of stuff
> > in separate processes today.  It appeared like only the first one
> > finished and/or the rest of the forked processes crashed.

> First thing to do would be to show us a little code and to look for the
> exit codes of your subprocess calls.

OK I've appended the 236 line Python script below that gets invoked
with Popen asynchronously.
In summary what it does is set up a web site with a hosting provider.
That involves using modules (Mechanize and Selenium) that literally
launch browsers and simulate mouse and keyboard actions to get the job
done.  Furthermore, Expect scripts are launched that copy code and SSH
to remote accounts to do other work.  Is any of that I just mentioned
especially troublesome in a Popen process?

# hosting company set up script

import sys
import os
sys.path.append(os.path.dirname(__file__) + "/../selenium")

import selenium
import mechanize
import subprocess
import smtplib
import re

HOSTMONSTER_URL = "https://www.hostmonster.com"
SELENIUM_PORT   = 4444
SUCCESS_DOMAIN  = "Successfully assigned _DOMAIN_ as addon domain"
SUCCESS_DB      = '<p>Added the database <span class="status">\w
+_DB_</sp'
TIMEOUT         = "120000"
WP_DIR          = os.path.dirname(__file__) + "/wordpress"

# Get the inputs.
user_name         = sys.argv[1]
cpanel_password   = sys.argv[2]
primary_domain    = sys.argv[3]
wp_admin_password = sys.argv[4]
wp_admin_email    = sys.argv[5]
wp_db_password    = sys.argv[6]
wp_akismet_key    = sys.argv[7]
wp_adsense_id     = sys.argv[8]
title             = sys.argv[9]
description       = sys.argv[10]
keywords          = sys.argv[11]
domain            = sys.argv[12]
post_title        = sys.argv[13]
post_text         = sys.argv[14]
theme             = sys.argv[15]
auto_log          = sys.argv[16]

# Initialize the output variable.
output = ""

# Initialize the Mechanize browser.
browser = mechanize.Browser()

# Perform authentication for the Mechanize browser.
browser.open(HOSTMONSTER_URL + "/cgi-bin/cplogin")
browser.select_form("l_login_form")
browser["ldomain"] = user_name
browser["lpass"]   = cpanel_password
response = browser.submit()
browser.open(response.geturl())
browser.select_form("the_form")
response = browser.submit()

# Go to the databases page.
browser.open(response.geturl())
link     = browser.find_link(text_regex = "^MySQL.*Databases$")
response = browser.follow_link(link)
browser.open(response.geturl())

# Create the database and log a report.
database = domain.replace(".", "")
browser.select_form("mainform")
browser["db"] = database
response      = browser.submit()
success_db    = SUCCESS_DB.replace("_DB_", database)
if re.search(success_db, response.get_data()):
        output += domain + " database creation: SUCCESS" + "\n"
else:
        output += domain + " database creation: FAIL" + "\n"
open(auto_log, "a").write(output.split("\n")[-2])

# Close the Mechanize browser.
browser.close()

# Initialize the Selenium browser.  (Hostmonster)
browser = selenium.selenium("localhost",
                            SELENIUM_PORT,
                            "*chrome",
                            HOSTMONSTER_URL)
browser.start()

# Perform authentication for the Selenium browser.  (HostMonster)
browser.open("/cgi/hosting/assign")
browser.type("ldomain", user_name)
browser.type("lpass",   cpanel_password)
browser.click("//input[@value = 'LOGIN']")
browser.wait_for_page_to_load(TIMEOUT)

# Assign the domain and log a report.
browser.type("domain", domain)
browser.type("dir",    domain)
browser.type("sub",    domain.replace(".", "-"))
browser.click("//input[@value = 'Add Domain']")
browser.wait_for_page_to_load(TIMEOUT)
success_domain = SUCCESS_DOMAIN.replace("_DOMAIN_", domain)
if success_domain in browser.get_html_source():
        output += domain + " domain assignment: SUCCESS" + "\n"
else:
        output += domain + " domain assignment: FAIL" + "\n"
open(auto_log, "a").write(output.split("\n")[-2])

# Close the Selenium browser.  (Hostmonster)
browser.stop()

# Initialize the WordPress instance and log a report.
expect_output  = subprocess.Popen([WP_DIR + "/wp_rsync_expect",
                                   user_name,
                                   cpanel_password,
                                   primary_domain],
                                  stdout = subprocess.PIPE,
                                  stderr =
subprocess.STDOUT).communicate()[0]
expect_output += subprocess.Popen([WP_DIR + "/wp_set_up_expect",
                                   user_name,
                                   cpanel_password,
                                   primary_domain,
                                   wp_db_password,
                                   domain,
                                   theme],
                                  stdout = subprocess.PIPE,
                                  stderr =
subprocess.STDOUT).communicate()[0]
if expect_output.strip().endswith("SUCCESS"):
        output += domain + " WordPress initialization: SUCCESS" + "\n"
else:
        output += domain + " WordPress initialization: FAIL" + "\n"
open(auto_log, "a").write(expect_output)
open(auto_log, "a").write(output.split("\n")[-2])

# Initialize the Selenium browser.  (WordPress instance)
browser = selenium.selenium("localhost",
                            SELENIUM_PORT,
                            "*chrome",
                            "http://" + domain)
browser.start()

# Set the WordPress instance title and admin email address.
browser.open("/wp-admin/install.php")
browser.type("weblog_title", title)
browser.type("admin_email", wp_admin_email)
browser.click("Submit")
browser.wait_for_page_to_load(TIMEOUT)

# Perform authentication for the Selenium browser.  (WordPress
instance)
temp_password = browser.get_html_source()
beg           = temp_password.find("<code>")
beg           = temp_password.find("<code>", beg + 1) + len("<code>")
end           = temp_password.find("</code>", beg)
temp_password = temp_password[beg:end]
browser.click("link=Log In")
browser.wait_for_page_to_load(TIMEOUT)
browser.type("user_login", "admin")
browser.type("user_pass", temp_password)
browser.click("wp-submit")
browser.wait_for_page_to_load(TIMEOUT)

# Change the admin password.
browser.open("/wp-admin/profile.php#password")
browser.type("pass1", wp_admin_password)
browser.type("pass2", wp_admin_password)
browser.click("submit")
browser.wait_for_page_to_load(TIMEOUT)

# Delete the default WordPress instance post.
browser.click("link=Posts")
browser.wait_for_page_to_load(TIMEOUT)
browser.click("link=Hello world!")
browser.wait_for_page_to_load(TIMEOUT)
browser.click("link=Move to Trash")
browser.wait_for_page_to_load(TIMEOUT)

# Add a WordPress instance post.
#browser.click("//div[@id = 'wpbody-content']/div[2]/h2/a")
#browser.wait_for_page_to_load(TIMEOUT)
#browser.type("title", post_title)
#browser.click("publish")
#browser.wait_for_page_to_load(TIMEOUT)

# Activate all the WordPress instance plugins.
browser.open("/wp-admin/plugins.php")
browser.wait_for_page_to_load(TIMEOUT)
browser.click("//input[@type = 'checkbox']")
browser.select("action", "label=Activate")
browser.click("doaction_active")
browser.wait_for_page_to_load(TIMEOUT)

# Configure the WordPress instance All In One SEO Pack plugin.
browser.click("link=Settings")
browser.wait_for_page_to_load(TIMEOUT)
browser.click("link=All in One SEO")
browser.wait_for_page_to_load(TIMEOUT)
browser.click("aiosp_enabled")
browser.type("aiosp_home_title", title)
browser.type("aiosp_home_description", description)
browser.type("aiosp_home_keywords", keywords)
browser.click("Submit")
browser.wait_for_page_to_load(TIMEOUT)

# Configure the WordPress instance Akismet plugin.
browser.click("link=enter your WordPress.com API key")
browser.wait_for_page_to_load(TIMEOUT)
browser.type("key", wp_akismet_key)
browser.click("akismet_discard_month")
browser.click("submit")
browser.wait_for_page_to_load(TIMEOUT)

# Configure the WordPress instance All In One Adsense And YPN! plugin.
browser.click("link=Adsense")
browser.wait_for_page_to_load(TIMEOUT)
browser.type("ai_client", wp_adsense_id)
browser.click("action")

# Generate the WordPress instance initial Google sitemap.
browser.click("link=XML-Sitemap")
browser.wait_for_page_to_load(TIMEOUT)
browser.click("link=Click here")
browser.wait_for_page_to_load(TIMEOUT)

# Set the WordPress instance theme.
browser.click("link=Appearance")
browser.wait_for_page_to_load(TIMEOUT)
browser.click("link=Activate")
browser.wait_for_page_to_load(TIMEOUT)

# Close the Selenium browser.  (WordPress instance)
browser.stop()

# XXXX Will need to improve this....log a report.
if True:
        output += domain + " WordPress configuration: SUCCESS" + "\n"
else:
        output += domain + " WordPress configuration: FAIL" + "\n"
open(auto_log, "a").write(output.split("\n")[-2])

# Email the results to the user.
email_handle = smtplib.SMTP("localhost")
email        = "Subject: " + domain + " report\n" + output
email_handle.sendmail("support at seoconquer.com", wp_admin_email, email)