Unicode

Sun Dec 16 16:10:37 EST 2012

Hello guys,

I'm using Linux CentOS and Python 2.4 with MySQL 5.xx, I get error
with Unicode I tried many things that I found on the net but none of
them working.

If I dont use UTF-8 it inserts the data into the DB  but some French
char. are not correctly decoded. Could you please help me ?

Thanks

def PrepareSpecs(product_id, icecat_prod_id, icecat_image_url, name):
"""Gets the specifications of a product from Icecat.biz and insert
them into the DB
"""
    specs = {3:GetSpecsNL(icecat_prod_id),2:GetSpecsFR(icecat_prod_id).decode('utf-8'),1:GetSpecsEN(icecat_prod_id)}
    SpecsToSQL(product_id,specs,name)
    CategorySQL(product_id)
    StoreSQL(product_id)
    GetIMG(icecat_image_url,icecat_prod_id)
    return

def GetSpecsFR(icecat_prod_id):
    opener = urllib.FancyURLopener({})
    ffr = opener.open("http://prf.icecat.biz/index.cgi?product_id=%s;mi=start;smi=product;shopname=openICEcat-url;lang=fr"
% icecat_prod_id)
    specsfr = ffr.read()
    #specsfr = specsfr.decode('utf-8')
    specsfr = RemoveHTML(specsfr)
    ##specsfr = "%r" % specsfr
##    if specsfr:
##        try:
##            specsfr = str(specsfr)
##        except UnicodeEncodeError:
##            specsfr = str(specsfr.encode('utf-16'))
    return specsfr

def RemoveHTML(specs):
    specs = specs.replace("<html>","")
    specs = specs.replace("<HTML>","")
    specs = specs.replace("</html>","")
    specs = specs.replace("</HTML>","")
    specs = specs.replace("<head>","")
    specs = specs.replace("<HEAD>","")
    specs = specs.replace("</head>","")
    specs = specs.replace("</HEAD>","")
    specs = specs.replace("<body>","")
    specs = specs.replace("</body>","")
    specs = specs.replace("<BODY>","")
    specs = specs.replace("</body>","")
    specs = specs.replace("<TITLE>","")
    specs = specs.replace("</TITLE>","")
    specs = specs.replace("<title>","")
    specs = specs.replace("</title>","")
    specs = specs.replace("<p>","")
    specs = specs.replace("</p>","")
    return specs

def SpecsToSQL(product_id, specs, name):
    for lang, spec in specs.iteritems():
        InsertSpecsDB(product_id, spec, lang, name)
    return

def InsertSpecsDB(product_id, spec, name, lang):
    db = MySQLdb.connect("localhost","getit","opencart")
    cursor = db.cursor()
    sql = "INSERT INTO product_description (product_id, language_id,
name, description) VALUES (%s,%s,%s,%s)"
    params = (product_id, lang, name, spec)
    cursor.execute(sql, params)
    id = cursor.lastrowid
    print"Updated ID %s description %s" %(int(id), lang)
    return