How do you dynamically load python classes from a given directory?

Question

If I define a module module with a corresponding directory of module/, can I dynamically load classes from children modules such as a.py or b.py?

--module
----a.py
----b.py

Does this require knowing the class name to search for? Could I setup a base class that will somehow load these children?

The basic use case is to allow a user to write some code of his or her own that the program will load in. The same as how rails allows you to write your own controllers, views, and models in certain directories.

The code for loading modules dynamically I have so far is

def load(folder):
    files = {}
    for filename in os.listdir(folder):
      if (filename[0] != '_' and filename[0] != '.'):
        files[filename.rstrip('.pyc')] = None

    # Append each module to the return list of modules
    modules = []
    mod = __import__(folder, fromlist=files.keys())
    for key in files.keys():
      modules.append(getattr(mod, key))

    return modules

I was hoping to modify it to return class objects.

Answer 1

You are looking for pkgutil.walk_packages. Using this you can do the following:

def load(root_import_path, is_valid=lambda entity: True):
    """Returns modules in ``root_import_path`` that satisfy the ``is_valid`` test

    :param root_import_path: An string name for importing (i.e. "myapp").
    :param is_valid: A callable that takes a variable and returns ``True``
                     if it is of interest to us."""

    prefix = root_import_path + u"."
    modules = []

    for _, name, is_pkg in walk_packages(root_import_path, prefix=prefix):
        if is_pkg: 
            continue
        module_code = __import__(name)
        contents = dir(module_code)
        for thing in contents:
            if is_valid(thing):
                modules.append(thing)

    return modules

Alternatly, if you don't mind taking on a dependency, you could try the straight.plugin loader, which is a little more complicated than this simple load function.

Answer 2

#!/usr/bin/env python

import os
import sys
import inspect

def load_modules_from_path(path):
   """
   Import all modules from the given directory
   """
   # Check and fix the path
   if path[-1:] != '/':
       path += '/'

   # Get a list of files in the directory, if the directory exists
   if not os.path.exists(path):
        raise OSError("Directory does not exist: %s" % path)

   # Add path to the system path
   sys.path.append(path)
   # Load all the files in path
   for f in os.listdir(path):
       # Ignore anything that isn't a .py file
       if len(f) > 3 and f[-3:] == '.py':
           modname = f[:-3]
           # Import the module
           __import__(modname, globals(), locals(), ['*'])

def load_class_from_name(fqcn):
    # Break apart fqcn to get module and classname
    paths = fqcn.split('.')
    modulename = '.'.join(paths[:-1])
    classname = paths[-1]
    # Import the module
    __import__(modulename, globals(), locals(), ['*'])
    # Get the class
    cls = getattr(sys.modules[modulename], classname)
    # Check cls
    if not inspect.isclass(cls):
       raise TypeError("%s is not a class" % fqcn)
    # Return class
    return cls

def main():
    load_modules_from_path('modules')
    # load the TestClass1
    class_name = load_class_from_name('class1.TestClass1')
    # instantiate the Testclass1
    obj = class_name()
    # using this object obj to call the attributes inside the class
    print obj.testclass1()

if __name__ == '__main__': main()

---

Inside modules directory, i ve another two modules for testing :

[♫ test] modules :~ pwd
/tmp/dynamic_loader/modules

[♫ test] modules :~ ls -lR
total 32
-rw-r--r--  1 staff  staff  138 Aug 30 21:10 class1.py
-rw-r--r--  1 staff  staff  575 Aug 30 21:11 class1.pyc
-rw-r--r--  1 staff  staff  139 Aug 30 21:11 class2.py
-rw-r--r--  1 staff  staff  576 Aug 30 21:11 class2.pyc

---

[♫ test] modules  cat class1.py

class TestClass1(object):
  def testclass1(self):
      print 'I am from testclass1'

  def some_function():
      print 'some function 1'