Hey, get this! [was: import from database]

[Steve Holden]

Steve Holden wrote:

> Peter Otten wrote:
> 
>> Steve Holden wrote:
>>
>>
>>> This is even stranger: it makes it if I import the module a second time:
>>
>>
>>
>> [second import seems to succeed]
>>
>> Maybe you are experiencing some version confusion? What you describe 
>> looks
>> much like the normal Python 2.3 behaviour (with no import hook involved)
>> whereas you seem to operate on the 2.4 library.
>> A partially initialized module object is left behind in sys.modules 
>> and seen
>> by further import attempts.
>>
> I agree that this is 2.3-like behavior, but Python cannot lie ...
> 
> sholden at dellboy ~/Projects/Python/dbimp
> $ python
> Python 2.4 (#1, Dec  4 2004, 20:10:33)
> [GCC 3.3.3 (cygwin special)] on cygwin
> Type "help", "copyright", "credits" or "license" for more information.
>  >>>
> 
Just to make things simpler, and (;-) to appeal to a wider audience, 
here is a program that doesn't use database at all (it loads the entire 
standard library into a dict) and still shows the error.

What *I* would like to know is: who is allowing the import of bsddb.os, 
thereby somehow causing the code of the os library module to be run a 
second time.

#
# Establish standard library in dict
#
import os
import glob
import sys
import marshal
import new

def importpy(dct, path, modname, package):
     c = compile(file(path).read(), path, "exec")
     dct[modname] = (marshal.dumps(c), package, path)
     if package:
         print "Package", modname, path
     else:
         print "Module", modname, path

def importall(dct, path, modlist):
     os.chdir(path)
     for f in glob.glob("*"):
         if os.path.isdir(f):
             fn = os.path.join(path, f, "__init__.py")
             if os.path.exists(fn):
                 ml = modlist + [f]
                 importpy(dct, fn, ".".join(ml), 1)
                 importall(dct, os.path.join(path, f), ml)
         elif f.endswith('.py') and '.' not in f[:-3] and f != 
"__init__.py":
             importpy(dct, os.path.join(path, f), 
".".join(modlist+[f[:-3]]), 0)

class dbimporter(object):

     def __init__(self, item, *args, **kw):
         ##print "dbimporter: item:", item, "args:", args, "keywords:", kw
         if item != "*db*":
             raise ImportError
         print "Accepted", item

     def find_module(self, fullname, path=None):
         print "find_module:", fullname, "from", path
         if fullname not in impdict:
             #print "Bailed on", fullname
             return None
         else:
             print "found", fullname, "in db"
             return self

     def load_module(self, modname):
         print "load_module:", modname
         if modname in sys.modules:
             return sys.modules[modname]
         try:
             row = impdict[modname]
         except KeyError:
             #print modname, "not found in db"
             raise ImportError, "DB module %s not found in modules"
         code, package, path = row
         code = marshal.loads(code)
         module = new.module(modname)
         sys.modules[modname] = module
         module.__name__ = modname
         module.__file__ = path # "db:%s" % modname
         module.__loader__ = dbimporter
         if package:
             module.__path__ = sys.path
         exec code in module.__dict__
         print modname, "loaded:", repr(module), "pkg:", package
         return module

def install():
     sys.path_hooks.append(dbimporter)
     sys.path_importer_cache.clear() # probably not necessary
     sys.path.insert(0, "*db*") # probably not needed with a metea-path 
hook?

if __name__ == "__main__":
     impdict = {}
     for path in sys.argv[1:]:
         importall(impdict, path, [])
     install()
     import bsddb

Running this against a copy of the Python 2.4 standard library in C:\Lib 
gives me

[...]
Module _strptime C:\Lib\_strptime.py
Module _threading_local C:\Lib\_threading_local.py
Module __future__ C:\Lib\__future__.py
Accepted *db*
find_module: bsddb from None
found bsddb in db
load_module: bsddb
find_module: bsddb._bsddb from None
find_module: bsddb.sys from None
find_module: bsddb.os from None
find_module: bsddb.nt from None
find_module: bsddb.ntpath from None
find_module: bsddb.stat from None
Traceback (most recent call last):
   File "C:\Steve\Projects\Python\dbimp\dictload.py", line 79, in ?
     import bsddb
   File "C:\Steve\Projects\Python\dbimp\dictload.py", line 65, in 
load_module
     exec code in module.__dict__
   File "C:\Lib\bsddb\__init__.py", line 62, in ?
     import sys, os
   File "C:\Python24\lib\os.py", line 133, in ?
     from os.path import (curdir, pardir, sep, pathsep, defpath, extsep, 
altsep,
ImportError: No module named path

The 2.3 bsddb library doesn't cause the same problems (and even loads 
into 2.4 quite nicely). Lots of modules *will* import, and most packages 
don't seem to cause problems. Anyone give me a pointer here?

regards
  Steve
-- 
Meet the Python developers and your c.l.py favorites March 23-25
Come to PyCon DC 2005          http://www.python.org/pycon/2005/
Steve Holden                           http://www.holdenweb.com/