• programming in python • a course for the curious •

Completion in Python Console

In this post I described how to turn on completion for Python console. Here I would like to publish a script which adds few more tricks with completion, which makes it even cooler than the completion in IPython:

  1. hide the special methods, and attributes, unless you specifically type object._<Tab>,

  2. complete module names for the import statement: the standard completion (i.e. the one from rlcopmleter) works for:

    >>> import o<Tab>
    

but not for:

>>> import os.p<Tab>

unless os module is already imported. In our case this will complete to os.path.

  1. Add completion for:

    >>> from  os import p<Tab>
    

This scripts violates security some issues, even more than the code in rlcompleter. The completion for various import statements requires that a module is imported. There are actually three separate cases. When you complete a name which does not have a . (dot): the list of modules is taken using the pkgutil.iter_modules - this is completely safe.  But when the `().`` is included, the parts before the last . is separated and the directory tree for that module is checked for its submodules. Also the __all__ variable is imported (and hence the module is executed). The third case is when you complete the statement from module import att<Tab>. In this case the module is imported and all its attributes are pulled using the dir() built-in function.

The completion for attributes of objects is taken from rlcompleter. If an object has __getattr__ hook it can execute arbitrary defined code.

You can download the script using this link.

from rlcompleter import Completer as _Completer
import __main__
import re
import time
import os
import os.path
import sys
import readline
import importlib

osdir = os.path.dirname(os.__file__)

__all__ = ["Completer"]

class Completer(_Completer):
    """
    This Completer class adds one method to Completer() class:
    self.mod_matches() and overwrites self.complete().
    """

    def __init__(self, namespace = None):
        _Completer.__init__(self, namespace)
        import pkgutil
        self.iter_modules = list(pkgutil.iter_modules())
        # somehow this *must* be unpacked to a list, otherwise the iterator is empty.

    def complete(self, text, state):
        """Return the next possible completion for 'text'.

        This is called successively with state == 0, 1, 2, ... until it returns
        None.  The completion should begin with 'text'.

        """

        line = readline.get_line_buffer()

        # This is usefull if you want to debug or play with it and see how it
        # works:
        self.text = text
        self.line = line

        if self.use_main_ns:
            self.namespace = __main__.__dict__

        if state == 0:
            if line.strip().startswith('import') or re.match(r'from\s+\S+$', line):
                """
                import docut<Tab> -> import docutils
                """
                self.state = 'import: import'
                self.matches = self.mod_matches(text, line)

            elif re.match(r'from\s+\S+\s+', line):
                if re.match(r'(from\s+\S+\s+)(?:i|im|imp|impo|impor)$', line):
                    """
                    from module i<Tab> -> from module import 
                    """
                    self.state = 'from: import'
                    self.matches=['import ']

                else:
                    """
                    from module import att<Tab> -> from module import attribute

                    Note: att cannot contain a dot:
                        from sphinx import writers.html
                    is not a valid Python syntax.
                    """
                    self.state = 'from: attr'
                    m = re.match('from\s+(\S+)\s+import\s+(.*)', line)
                    module = m.group(1)
                    self.module = module
                    matches = self.get_module_attrs(module)
                    matches = [ match for match in matches if match.startswith(text)]
                    self.matches = matches

            elif "." in text:
                """
                object.att<Tab> -> object.attribute
                """
                self.state = 'obj: attr'
                self.matches = self.attr_matches(text)
                # filter completions if attribute doesn't start with a '_':
                if not re.match(r"(\w+(\.\w+)*)\.(\w*)", text).group(3).startswith("_"):
                    self.matches = filter(lambda m: not re.match(r"(\w+(\.\w+)*)\.(\w*)", m).group(3).startswith("_"), self.matches)
                self.matches = [ match for match in self.matches ]

            else:
                """
                ob<Tab> -> object
                """
                self.state = 'obj'
                self.matches = self.global_matches(text)
                # filter completions if text doesn't begin with a '_':
                if not text.startswith("_"):
                    self.matches = filter(lambda m: not m.startswith("_"), self.matches)
                self.matches = [ match for match in self.matches ]
        try:
            return self.matches[state]
        except IndexError:
            return None

    def get_module_attrs(self, module_name):
        """Return module attributes of module `module_name`.

        Filtering is not done here.
        """
        module = importlib.import_module(module_name)
        matches = dir(module)
        del module
        return set(matches)

    def filter(self,package,dirname):
        if os.path.isdir(os.path.join(dirname, package)):
            return os.path.isfile(os.path.join(dirname, package, '__init__.py'))
        else:
            return package.endswith('.py')

    def get_submodules(self, module, smod_list=[]):
        # This returns to much, for example it returns os.sys (since os imports sys)
        # mod = importlib.import_module(module)
        # submods = [ submod for submod in dir(mod) if type(getattr(mod, submod)) == type(mod)]
        # del mod
        # return submodules
        global osdir
        base = module
        try:
            _temp = __import__(base, [], [], ['__file__'])
        except ImportError as e:
            base = '.'.join(smod_list[:-2])
            _temp = __import__(base, [], [], ['__file__'])
            file = _temp.__file__
            del _temp

            dirname = os.path.dirname(file)
            if os.path.isfile(os.path.join(dirname, smod_list[-2], '__init__.py')):
                file = os.path.join(dirname, smod_list[-2], '__init__.py')
            else:
                """
                There is no reason to try .py file since we would have
                to load it to get the list of submodule, and loading has
                just failed.
                """
                return []
            dirname = os.path.dirname(file)
        else:
            file = _temp.__file__
            dirname = os.path.dirname(file)
        if dirname == osdir or os.path.splitext(os.path.basename(file))[0] != '__init__':
            """
            If the base module is a standard module or it is a file, then
            we use __all__.  In this way we avoid things like (os.sys, ...).
            """
            _temp = __import__(base, [], [], '__all__')

            matches = []
            if hasattr(_temp, '__all__'):
                for attrn in _temp.__all__:
                    if type(getattr(_temp, attrn)) == type(_temp):
                        matches.append(attrn)
            del _temp
            return matches
        else:
            matches = os.listdir(dirname)
            matches = filter(lambda p: self.filter(p,dirname), matches)
            matches = map(lambda p: os.path.splitext(p)[0], matches)
            matches = set(matches)
            if '__init__' in matches:
                matches.remove('__init__')
            return matches

    def mod_matches(self, text, line):
        """
        Return a list of matching packages for the import statement.

        We are reading the __all__ and __file__ attributes, and traverse the
        directory structure to get all the submodules (self.get_submodules).
        """

        if '.' in text:
            start =  '.'.join(text.split('.')[:-1])
        else:
            start = ''
        self.text = text
        self.start = start

        matches = []
        if "." in text:
            """
            module.submodu<Tab> -> list of matching submodules
            """
            smod_list = text.split(".")
            smod = smod_list[-1]
            base = '.'.join(smod_list[:-1])
            _matches = self.get_submodules(base)
            for match in _matches:
                if match.startswith(smod):
                    matches.append("%s.%s" % (start, match))
            return matches
        else:
            """
            modu<Tab> -> list of matching modules
            """
            for (module, match, ispkg) in self.iter_modules:
                if match.startswith(text) and (len(text) == 0 and not text.startswith("_") or len(text)):
                    matches.append(match)
            return matches

try:
    import readline
    readline.set_completer_delims('\t \n`~!@#$%^&*()-=+[{]}\\|;:\'",<>/?')
except ImportError as e:
    print(e)
else:
    completer = Completer()
    readline.set_completer(completer.complete)
    # If you want to debug the completer you can always import readline module, then
    # completer = readline.get_completer().im_self is the defined completer object.
Last update: 2013-11-06 11:02 (CET)
Contact us.
Created using , and
we are not associated with Python Software Foundation
© Copyright 2012, 2013 Accorda Institute.