Merged revisions 61598-61599,61601 via svnmerge from

svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3

........
  r61598 | david.wolever | 2008-03-18 23:58:33 -0500 (Di, 18 Mär 2008) | 1 line

  Added fixer for zip, and refactored a bit of code in the process.  Closing #2171.
........
  r61599 | david.wolever | 2008-03-19 00:04:26 -0500 (Mi, 19 Mär 2008) | 3 lines

  Removed a bunch of duplicate code -- it's in util now.
........
  r61601 | martin.v.loewis | 2008-03-19 00:21:12 -0500 (Mi, 19 Mär 2008) | 2 lines

  Fix whitespace.
........

Lib/lib2to3/fixes/basefix.py

@@ -87,7 +87,7 @@ def transform(self, node, results):
         Args:
           node: the root of the parse tree that matched the fixer.
           results: a dict mapping symbolic names to part of the match.
-          
+
         Returns:
           None, or a node that is a modified copy of the
           argument node.  The node argument may also be modified in-place to
@@ -146,7 +146,7 @@ def warning(self, node, reason):
     def start_tree(self, tree, filename):
         """Some fixers need to maintain tree-wide state.
         This method is called once, at the start of tree fix-up.
-        
+
         tree - the root node of the tree to be processed.
         filename - the name of the file the tree came from.
         """
@@ -158,7 +158,7 @@ def start_tree(self, tree, filename):
     def finish_tree(self, tree, filename):
         """Some fixers need to maintain tree-wide state.
         This method is called once, at the conclusion of tree fix-up.
-        
+
         tree - the root node of the tree to be processed.
         filename - the name of the file the tree came from.
         """

Lib/lib2to3/fixes/fix_except.py

@@ -3,19 +3,19 @@
 The following cases will be converted:
 
 - "except E, T:" where T is a name:
-    
+
     except E as T:
-    
+
 - "except E, T:" where T is not a name, tuple or list:
-    
+
         except E as t:
             T = t
-        
+
     This is done because the target of an "except" clause must be a
     name.
-    
+
 - "except E, T:" where T is a tuple or list literal:
-    
+
         except E as t:
             T = t.args
 """
@@ -39,7 +39,7 @@ class FixExcept(basefix.BaseFix):
     try_stmt< 'try' ':' suite
                   cleanup=((except_clause ':' suite)+ ['else' ':' suite]
                                                       ['finally' ':' suite]
-	                       | 'finally' ':' suite) >
+                               | 'finally' ':' suite) >
     """
 
     def transform(self, node, results):

Lib/lib2to3/fixes/fix_filter.py

@@ -14,11 +14,9 @@
 """
 
 # Local imports
-from .. import pytree
-from .. import patcomp
 from ..pgen2 import token
 from . import basefix
-from .util import Name, Call, ListComp, attr_chain, does_tree_import
+from .util import Name, Call, ListComp, does_tree_import, in_special_context
 
 class FixFilter(basefix.BaseFix):
 
@@ -85,35 +83,3 @@ def transform(self, node, results):
             new = Call(Name("list"), [new])
         new.set_prefix(node.get_prefix())
         return new
-
-P0 = """for_stmt< 'for' any 'in' node=any ':' any* >
-        | comp_for< 'for' any 'in' node=any any* >
-     """
-p0 = patcomp.compile_pattern(P0)
-
-P1 = """
-power<
-    ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' |
-      'any' | 'all' | (any* trailer< '.' 'join' >) )
-    trailer< '(' node=any ')' >
-    any*
->
-"""
-p1 = patcomp.compile_pattern(P1)
-
-P2 = """
-power<
-    'sorted'
-    trailer< '(' arglist<node=any any*> ')' >
-    any*
->
-"""
-p2 = patcomp.compile_pattern(P2)
-
-def in_special_context(node):
-    patterns = [p0, p1, p2]
-    for pattern, parent in zip(patterns, attr_chain(node, "parent")):
-        results = {}
-        if pattern.match(parent, results) and results["node"] is node:
-            return True
-    return False

Lib/lib2to3/fixes/fix_future.py

@@ -2,15 +2,14 @@
 
 from __future__ import foo is replaced with an empty line.
 """
-# Author: Christian Heimes 
+# Author: Christian Heimes
 
 # Local imports
 from . import basefix
-from .util import BlankLine 
+from .util import BlankLine
 
 class FixFuture(basefix.BaseFix):
     PATTERN = """import_from< 'from' module_name="__future__" 'import' any >"""
 
     def transform(self, node, results):
         return BlankLine()
-

Lib/lib2to3/fixes/fix_has_key.py

@@ -18,14 +18,14 @@
     m = d.has_key
     if m(k):
         ...
-        
+
    Only *calls* to has_key() are converted. While it is possible to
    convert the above to something like
-   
+
     m = d.__contains__
     if m(k):
         ...
-        
+
    this is currently not done.
 """
 

Lib/lib2to3/fixes/fix_idioms.py

@@ -5,7 +5,7 @@
     type(x) is T -> isinstance(x, T)
     type(x) != T -> not isinstance(x, T)
     type(x) is not T -> not isinstance(x, T)
-    
+
 * Change "while 1:" into "while True:".
 
 * Change both
@@ -19,7 +19,7 @@
     v = EXPR
     v.sort()
     foo(v)
-    
+
 into
 
     v = sorted(EXPR)

Lib/lib2to3/fixes/fix_imports.py

@@ -16,7 +16,7 @@
 
 MAPPING = {"StringIO":  ("io", ["StringIO"]),
            "cStringIO": ("io", ["StringIO"]),
-           "__builtin__" : ("builtins", builtin_names), 
+           "__builtin__" : ("builtins", builtin_names),
           }
 
 
@@ -86,4 +86,4 @@ def transform(self, node, results):
             bare_name = bare_name[0]
             new_name = self.replace.get(bare_name.value)
             if new_name:
-              bare_name.replace(Name(new_name, prefix=bare_name.get_prefix()))
+                bare_name.replace(Name(new_name, prefix=bare_name.get_prefix()))

Lib/lib2to3/fixes/fix_map.py

@@ -20,11 +20,9 @@
 """
 
 # Local imports
-from .. import pytree
-from .. import patcomp
 from ..pgen2 import token
 from . import basefix
-from .util import Name, Call, ListComp, attr_chain, does_tree_import
+from .util import Name, Call, ListComp, does_tree_import, in_special_context
 from ..pygram import python_symbols as syms
 
 class FixMap(basefix.BaseFix):
@@ -71,7 +69,7 @@ def transform(self, node, results):
             # If a future map has been imported for this file, we won't
             # be making any modifications
             return
-            
+
         if node.parent.type == syms.simple_stmt:
             self.warning(node, "You should use a for loop here")
             new = node.clone()
@@ -92,35 +90,3 @@ def transform(self, node, results):
             new = Call(Name("list"), [new])
         new.set_prefix(node.get_prefix())
         return new
-
-P0 = """for_stmt< 'for' any 'in' node=any ':' any* >
-        | comp_for< 'for' any 'in' node=any any* >
-     """
-p0 = patcomp.compile_pattern(P0)
-
-P1 = """
-power<
-    ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' |
-      'any' | 'all' | (any* trailer< '.' 'join' >) )
-    trailer< '(' node=any ')' >
-    any*
->
-"""
-p1 = patcomp.compile_pattern(P1)
-
-P2 = """
-power<
-    'sorted'
-    trailer< '(' arglist<node=any any*> ')' >
-    any*
->
-"""
-p2 = patcomp.compile_pattern(P2)
-
-def in_special_context(node):
-    patterns = [p0, p1, p2]
-    for pattern, parent in zip(patterns, attr_chain(node, "parent")):
-        results = {}
-        if pattern.match(parent, results) and results["node"] is node:
-            return True
-    return False

Lib/lib2to3/fixes/fix_ne.py

@@ -17,6 +17,6 @@ def match(self, node):
         return node.type == token.NOTEQUAL and node.value == "<>"
 
     def transform(self, node, results):
-      new = pytree.Leaf(token.NOTEQUAL, "!=")
+        new = pytree.Leaf(token.NOTEQUAL, "!=")
-      new.set_prefix(node.get_prefix())
+        new.set_prefix(node.get_prefix())
-      return new
+        return new

Lib/lib2to3/fixes/fix_next.py

@@ -47,12 +47,12 @@ def transform(self, node, results):
         mod = results.get("mod")
 
         if base:
-          if self.shadowed_next:
+            if self.shadowed_next:
-              attr.replace(Name("__next__", prefix=attr.get_prefix()))
+                attr.replace(Name("__next__", prefix=attr.get_prefix()))
-          else:
+            else:
-              base = [n.clone() for n in base]
+                base = [n.clone() for n in base]
-              base[0].set_prefix("")
+                base[0].set_prefix("")
-              node.replace(Call(Name("next", prefix=node.get_prefix()), base))
+                node.replace(Call(Name("next", prefix=node.get_prefix()), base))
         elif name:
             n = Name("__next__", prefix=name.get_prefix())
             name.replace(n)

Lib/lib2to3/fixes/fix_print.py

@@ -5,7 +5,7 @@
 
 Change:
     'print'          into 'print()'
-    'print ...'	     into 'print(...)'
+    'print ...'      into 'print(...)'
     'print ... ,'    into 'print(..., end=" ")'
     'print >>x, ...' into 'print(..., file=x)'
 """

Lib/lib2to3/fixes/fix_raise.py

@@ -12,9 +12,9 @@
 CAVEATS:
 1) "raise E, V" will be incorrectly translated if V is an exception
    instance. The correct Python 3 idiom is
-   
+
         raise E from V
-        
+
    but since we can't detect instance-hood by syntax alone and since
    any client code would have to be changed as well, we don't automate
    this.
@@ -48,11 +48,11 @@ def transform(self, node, results):
         # Since Python 3 will not support this, we recurse down any tuple
         # literals, always taking the first element.
         if is_tuple(exc):
-          while is_tuple(exc):
+            while is_tuple(exc):
-              # exc.children[1:-1] is the unparenthesized tuple
+                # exc.children[1:-1] is the unparenthesized tuple
-              # exc.children[1].children[0] is the first element of the tuple
+                # exc.children[1].children[0] is the first element of the tuple
-              exc = exc.children[1].children[0].clone()
+                exc = exc.children[1].children[0].clone()
-          exc.set_prefix(" ")
+            exc.set_prefix(" ")
 
         if "val" not in results:
             # One-argument raise

Lib/lib2to3/fixes/fix_renames.py

@@ -63,8 +63,7 @@ def transform(self, node, results):
         attr_name = results.get("attr_name")
         #bare_name = results.get("bare_name")
         #import_mod = results.get("module")
-      
+
         if mod_name and attr_name:
             new_attr = LOOKUP[(mod_name.value, attr_name.value)]
             attr_name.replace(Name(new_attr, prefix=attr_name.get_prefix()))
-

Lib/lib2to3/fixes/fix_repr.py

@@ -15,8 +15,8 @@ class FixRepr(basefix.BaseFix):
               """
 
     def transform(self, node, results):
-      expr = results["expr"].clone()
+        expr = results["expr"].clone()
 
-      if expr.type == self.syms.testlist1:
+        if expr.type == self.syms.testlist1:
-          expr = self.parenthesize(expr)
+            expr = self.parenthesize(expr)
-      return Call(Name("repr"), [expr], prefix=node.get_prefix())
+        return Call(Name("repr"), [expr], prefix=node.get_prefix())

Lib/lib2to3/fixes/fix_tuple_params.py

@@ -2,7 +2,7 @@
 
 def func(((a, b), c), d):
     ...
-    
+
     ->
 
 def func(x, d):
@@ -10,7 +10,7 @@ def func(x, d):
     ...
 
 It will also support lambdas:
-  
+
     lambda (x, y): x + y -> lambda t: t[0] + t[1]
 
     # The parens are a syntax error in Python 3

Lib/lib2to3/fixes/fix_unicode.py

@@ -8,21 +8,21 @@
 
 class FixUnicode(basefix.BaseFix):
 
-  PATTERN = "STRING | NAME<'unicode' | 'unichr'>"
+    PATTERN = "STRING | NAME<'unicode' | 'unichr'>"
 
-  def transform(self, node, results):
+    def transform(self, node, results):
-    if node.type == token.NAME:
+        if node.type == token.NAME:
-      if node.value == "unicode":
+            if node.value == "unicode":
-        new = node.clone()
+                new = node.clone()
-        new.value = "str"
+                new.value = "str"
-        return new
+                return new
-      if node.value == "unichr":
+            if node.value == "unichr":
-        new = node.clone()
+                new = node.clone()
-        new.value = "chr"
+                new.value = "chr"
-        return new
+                return new
-      # XXX Warn when __unicode__ found?
+            # XXX Warn when __unicode__ found?
-    elif node.type == token.STRING:
+        elif node.type == token.STRING:
-      if re.match(r"[uU][rR]?[\'\"]", node.value):
+            if re.match(r"[uU][rR]?[\'\"]", node.value):
-        new = node.clone()
+                new = node.clone()
-        new.value = new.value[1:]
+                new.value = new.value[1:]
-        return new
+                return new

Lib/lib2to3/fixes/fix_ws_comma.py

@@ -11,29 +11,29 @@
 
 class FixWsComma(basefix.BaseFix):
 
-  explicit = True # The user must ask for this fixers
+    explicit = True # The user must ask for this fixers
 
-  PATTERN = """
+    PATTERN = """
-  any<(not(',') any)+ ',' ((not(',') any)+ ',')* [not(',') any]>
+    any<(not(',') any)+ ',' ((not(',') any)+ ',')* [not(',') any]>
-  """
+    """
 
-  COMMA = pytree.Leaf(token.COMMA, ",")
+    COMMA = pytree.Leaf(token.COMMA, ",")
-  COLON = pytree.Leaf(token.COLON, ":")
+    COLON = pytree.Leaf(token.COLON, ":")
-  SEPS = (COMMA, COLON)
+    SEPS = (COMMA, COLON)
 
-  def transform(self, node, results):
+    def transform(self, node, results):
-    new = node.clone()
+        new = node.clone()
-    comma = False
-    for child in new.children:
-      if child in self.SEPS:
-        prefix = child.get_prefix()
-        if prefix.isspace() and "\n" not in prefix:
-          child.set_prefix("")
-        comma = True
-      else:
-        if comma:
-          prefix = child.get_prefix()
-          if not prefix:
-            child.set_prefix(" ")
         comma = False
-    return new
+        for child in new.children:
+            if child in self.SEPS:
+                prefix = child.get_prefix()
+                if prefix.isspace() and "\n" not in prefix:
+                    child.set_prefix("")
+                comma = True
+            else:
+                if comma:
+                    prefix = child.get_prefix()
+                    if not prefix:
+                        child.set_prefix(" ")
+                comma = False
+        return new

Lib/lib2to3/fixes/fix_zip.py

@@ -0,0 +1,43 @@
+"""
+Fixer that changes zip(seq0, seq1, ...) into list(zip(seq0, seq1, ...)
+unless there exists a 'from future_builtins import zip' statement in the
+top-level namespace.
+
+We avoid the transformation if the zip() call is directly contained in
+iter(<>), list(<>), tuple(<>), sorted(<>), ...join(<>), or for V in <>:.
+"""
+
+# Local imports
+from . import basefix
+from .util import Name, Call, does_tree_import, in_special_context
+
+class FixZip(basefix.BaseFix):
+
+    PATTERN = """
+    power< 'zip' args=trailer< '(' [any] ')' >
+    >
+    """
+
+    def start_tree(self, *args):
+        super(FixZip, self).start_tree(*args)
+        self._future_zip_found = None
+
+    def has_future_zip(self, node):
+        if self._future_zip_found is not None:
+            return self._future_zip_found
+        self._future_zip_found = does_tree_import('future_builtins', 'zip', node)
+        return self._future_zip_found
+
+    def transform(self, node, results):
+        if self.has_future_zip(node):
+            # If a future zip has been imported for this file, we won't
+            # be making any modifications
+            return
+
+        if in_special_context(node):
+            return None
+        new = node.clone()
+        new.set_prefix("")
+        new = Call(Name("list"), [new])
+        new.set_prefix(node.get_prefix())
+        return new

Lib/lib2to3/fixes/util.py

@@ -5,6 +5,7 @@
 from ..pgen2 import token
 from ..pytree import Leaf, Node
 from ..pygram import python_symbols as syms
+from .. import patcomp
 
 
 ###########################################################
@@ -163,15 +164,15 @@ def reversed(l):
 
 def attr_chain(obj, attr):
     """Follow an attribute chain.
-    
+
     If you have a chain of objects where a.foo -> b, b.foo-> c, etc,
     use this to iterate over all objects in the chain. Iteration is
     terminated by getattr(x, attr) is None.
-    
+
     Args:
         obj: the starting object
         attr: the name of the chaining attribute
-    
+
     Yields:
         Each successive object in the chain.
     """
@@ -180,6 +181,44 @@ def attr_chain(obj, attr):
         yield next
         next = getattr(next, attr)
 
+p0 = """for_stmt< 'for' any 'in' node=any ':' any* >
+        | comp_for< 'for' any 'in' node=any any* >
+     """
+p1 = """
+power<
+    ( 'iter' | 'list' | 'tuple' | 'sorted' | 'set' | 'sum' |
+      'any' | 'all' | (any* trailer< '.' 'join' >) )
+    trailer< '(' node=any ')' >
+    any*
+>
+"""
+p2 = """
+power<
+    'sorted'
+    trailer< '(' arglist<node=any any*> ')' >
+    any*
+>
+"""
+pats_built = False
+def in_special_context(node):
+    """ Returns true if node is in an environment where all that is required
+        of it is being itterable (ie, it doesn't matter if it returns a list
+        or an itterator).
+        See test_map_nochange in test_fixers.py for some examples and tests.
+        """
+    global p0, p1, p2, pats_built
+    if not pats_built:
+        p1 = patcomp.compile_pattern(p1)
+        p0 = patcomp.compile_pattern(p0)
+        p2 = patcomp.compile_pattern(p2)
+        pats_built = True
+    patterns = [p0, p1, p2]
+    for pattern, parent in zip(patterns, attr_chain(node, "parent")):
+        results = {}
+        if pattern.match(parent, results) and results["node"] is node:
+            return True
+    return False
+
 ###########################################################
 ### The following functions are to find bindings in a suite
 ###########################################################
@@ -240,8 +279,8 @@ def find_binding(name, node, package=None):
         elif child.type == syms.simple_stmt:
             ret = find_binding(name, child, package)
         elif child.type == syms.expr_stmt:
-                if _find(name, child.children[0]):
+            if _find(name, child.children[0]):
-                    ret = child
+                ret = child
 
         if ret:
             if not package:

Lib/lib2to3/refactor.py

@@ -118,7 +118,7 @@ def __init__(self, options):
 
     def get_fixers(self):
         """Inspects the options to load the requested patterns and handlers.
-        
+
         Returns:
           (pre_order, post_order), where pre_order is the list of fixers that
           want a pre-order AST traversal, and post_order is the list that want
@@ -232,11 +232,11 @@ def refactor_file(self, filename):
 
     def refactor_string(self, data, name):
         """Refactor a given input string.
-        
+
         Args:
             data: a string holding the code to be refactored.
             name: a human-readable name for use in error/log messages.
-            
+
         Returns:
             An AST corresponding to the refactored input stream; None if
             there were errors during the parse.
@@ -274,12 +274,12 @@ def refactor_stdin(self):
 
     def refactor_tree(self, tree, name):
         """Refactors a parse tree (modifying the tree in place).
-        
+
         Args:
             tree: a pytree.Node instance representing the root of the tree
                   to be refactored.
             name: a human-readable name for this tree.
-        
+
         Returns:
             True if the tree was modified, False otherwise.
         """
@@ -296,13 +296,13 @@ def refactor_tree(self, tree, name):
 
     def traverse_by(self, fixers, traversal):
         """Traverse an AST, applying a set of fixers to each node.
-        
+
         This is a helper method for refactor_tree().
-        
+
         Args:
             fixers: a list of fixer instances.
             traversal: a generator that yields AST nodes.
-        
+
         Returns:
             None
         """
@@ -523,4 +523,4 @@ def diff_texts(a, b, filename):
 
 
 if __name__ == "__main__":
-  sys.exit(main())
+    sys.exit(main())

Lib/lib2to3/tests/benchmark.py

@@ -27,10 +27,10 @@ def __init__(self, **kwargs):
             setattr(self, k, v)
 
         self.verbose = False
-        
+
 def dummy_transform(*args, **kwargs):
     pass
-    
+
 ### Collect list of modules to match against
 ###############################################################################
 files = []

Lib/lib2to3/tests/test_fixers.py

@@ -2493,6 +2493,69 @@ def test_future_builtins(self):
         a = "from future_builtins import *; map(f, 'ham')"
         self.unchanged(a)
 
+class Test_zip(FixerTestCase):
+    fixer = "zip"
+
+    def check(self, b, a):
+        self.unchanged("from future_builtins import zip; " + b, a)
+        FixerTestCase.check(self, b, a)
+
+    def test_zip_basic(self):
+        b = """x = zip(a, b, c)"""
+        a = """x = list(zip(a, b, c))"""
+        self.check(b, a)
+
+        b = """x = len(zip(a, b))"""
+        a = """x = len(list(zip(a, b)))"""
+        self.check(b, a)
+
+    def test_zip_nochange(self):
+        a = """b.join(zip(a, b))"""
+        self.unchanged(a)
+        a = """(a + foo(5)).join(zip(a, b))"""
+        self.unchanged(a)
+        a = """iter(zip(a, b))"""
+        self.unchanged(a)
+        a = """list(zip(a, b))"""
+        self.unchanged(a)
+        a = """list(zip(a, b))[0]"""
+        self.unchanged(a)
+        a = """set(zip(a, b))"""
+        self.unchanged(a)
+        a = """set(zip(a, b)).pop()"""
+        self.unchanged(a)
+        a = """tuple(zip(a, b))"""
+        self.unchanged(a)
+        a = """any(zip(a, b))"""
+        self.unchanged(a)
+        a = """all(zip(a, b))"""
+        self.unchanged(a)
+        a = """sum(zip(a, b))"""
+        self.unchanged(a)
+        a = """sorted(zip(a, b))"""
+        self.unchanged(a)
+        a = """sorted(zip(a, b), key=blah)"""
+        self.unchanged(a)
+        a = """sorted(zip(a, b), key=blah)[0]"""
+        self.unchanged(a)
+        a = """for i in zip(a, b): pass"""
+        self.unchanged(a)
+        a = """[x for x in zip(a, b)]"""
+        self.unchanged(a)
+        a = """(x for x in zip(a, b))"""
+        self.unchanged(a)
+
+    def test_future_builtins(self):
+        a = "from future_builtins import spam, zip, eggs; zip(a, b)"
+        self.unchanged(a)
+
+        b = """from future_builtins import spam, eggs; x = zip(a, b)"""
+        a = """from future_builtins import spam, eggs; x = list(zip(a, b))"""
+        self.check(b, a)
+
+        a = "from future_builtins import *; zip(a, b)"
+        self.unchanged(a)
+
 class Test_standarderror(FixerTestCase):
     fixer = "standarderror"
 

Lib/lib2to3/tests/test_parser.py

@@ -109,7 +109,7 @@ def test_old(self):
         self.validate(s)
 
 
-# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms       
+# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
 class TestSetLiteral(GrammarTest):
     def test_1(self):
         self.validate("""x = {'one'}""")

Lib/lib2to3/tests/test_pytree.py

@@ -407,10 +407,10 @@ def testGenerateMatches(self):
         pe = pytree.LeafPattern(1, "e", "pe")
         pf = pytree.LeafPattern(1, "f", "pf")
         pw = pytree.WildcardPattern([[pa, pb, pc], [pd, pe],
-	                             [pa, pb], [pc, pd], [pe, pf]],
+                                     [pa, pb], [pc, pd], [pe, pf]],
                                     min=1, max=4, name="pw")
         self.assertEqual([x[0] for x in pw.generate_matches(leaves)],
-	                 [3, 5, 2, 4, 6])
+                         [3, 5, 2, 4, 6])
         pr = pytree.NodePattern(type=1000, content=[pw], name="pr")
         matches = list(pytree.generate_matches([pr], [root]))
         self.assertEqual(len(matches), 1)