Merged revisions 74114 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk ................ r74114 | benjamin.peterson | 2009-07-20 10:33:09 -0500 (Mon, 20 Jul 2009) | 110 lines Merged revisions 73771,73811,73840,73842,73848-73849,73861,73957-73960,73964-73969,73972-73974,73977,73981,73984,74065,74113 via svnmerge from svn+ssh://pythondev@svn.python.org/sandbox/trunk/2to3/lib2to3 ........ r73771 | benjamin.peterson | 2009-07-02 10:56:55 -0500 (Thu, 02 Jul 2009) | 1 line force the imports fixer to be run after the import one python#6400 ........ r73811 | benjamin.peterson | 2009-07-03 09:03:14 -0500 (Fri, 03 Jul 2009) | 1 line check for sep, not pathsep when looking for a subpackage python#6408 ........ r73840 | benjamin.peterson | 2009-07-04 09:52:28 -0500 (Sat, 04 Jul 2009) | 1 line don't print diffs by default; it's annoying ........ r73842 | benjamin.peterson | 2009-07-04 09:58:46 -0500 (Sat, 04 Jul 2009) | 1 line complain when not showing diffs or writing ........ r73848 | alexandre.vassalotti | 2009-07-04 23:38:19 -0500 (Sat, 04 Jul 2009) | 2 lines Fix test_refactor_stdin to handle print_output() method with 4 arguments. ........ r73849 | alexandre.vassalotti | 2009-07-04 23:43:18 -0500 (Sat, 04 Jul 2009) | 5 lines Issue 2370: Add fixer for the removal of operator.isCallable() and operator.sequenceIncludes(). Patch contributed by Jeff Balogh (and updated by me). ........ r73861 | benjamin.peterson | 2009-07-05 09:15:53 -0500 (Sun, 05 Jul 2009) | 1 line cleanup and use unicode where appropiate ........ r73957 | benjamin.peterson | 2009-07-11 15:49:56 -0500 (Sat, 11 Jul 2009) | 1 line fix calls to str() with unicode() ........ r73958 | benjamin.peterson | 2009-07-11 15:51:51 -0500 (Sat, 11 Jul 2009) | 1 line more str() -> unicode() ........ r73959 | benjamin.peterson | 2009-07-11 16:40:08 -0500 (Sat, 11 Jul 2009) | 1 line add tests for refactor_dir() ........ r73960 | benjamin.peterson | 2009-07-11 16:44:32 -0500 (Sat, 11 Jul 2009) | 1 line don't parse files just because they end with 'py' (no dot) ........ r73964 | benjamin.peterson | 2009-07-11 17:30:15 -0500 (Sat, 11 Jul 2009) | 1 line simplify ........ r73965 | benjamin.peterson | 2009-07-11 17:31:30 -0500 (Sat, 11 Jul 2009) | 1 line remove usage of get_prefix() ........ r73966 | benjamin.peterson | 2009-07-11 17:33:35 -0500 (Sat, 11 Jul 2009) | 1 line revert unintended change in 73965 ........ r73967 | benjamin.peterson | 2009-07-11 17:34:44 -0500 (Sat, 11 Jul 2009) | 1 line avoid expensive checks and assume the node did change ........ r73968 | benjamin.peterson | 2009-07-11 20:46:46 -0500 (Sat, 11 Jul 2009) | 1 line use a regular dict for the heads to avoid adding lists in the loop ........ r73969 | benjamin.peterson | 2009-07-11 20:50:43 -0500 (Sat, 11 Jul 2009) | 1 line prefix headnode functions with '_' ........ r73972 | benjamin.peterson | 2009-07-11 21:25:45 -0500 (Sat, 11 Jul 2009) | 1 line try to make the head node dict as sparse as possible ........ r73973 | benjamin.peterson | 2009-07-11 21:59:49 -0500 (Sat, 11 Jul 2009) | 1 line a better idea; add an option to *not* print diffs ........ r73974 | benjamin.peterson | 2009-07-11 22:00:29 -0500 (Sat, 11 Jul 2009) | 1 line add space ........ r73977 | benjamin.peterson | 2009-07-12 10:16:07 -0500 (Sun, 12 Jul 2009) | 1 line update get_headnode_dict tests for recent changes ........ r73981 | benjamin.peterson | 2009-07-12 12:06:39 -0500 (Sun, 12 Jul 2009) | 4 lines detect when "from __future__ import print_function" is given Deprecate the 'print_function' option and the -p flag ........ r73984 | benjamin.peterson | 2009-07-12 16:16:37 -0500 (Sun, 12 Jul 2009) | 1 line add tests for Call; thanks Joe Amenta ........ r74065 | benjamin.peterson | 2009-07-17 12:52:49 -0500 (Fri, 17 Jul 2009) | 1 line pathname2url and url2pathname are in urllib.request not urllib.parse python#6496 ........ r74113 | benjamin.peterson | 2009-07-20 08:56:57 -0500 (Mon, 20 Jul 2009) | 1 line fix deprecation warnings in tests ........ ................
asvetlov · Jul 20, 2009 · 3059b00 · 3059b00
1 parent 6919427
commit 3059b00
Show file tree

Hide file tree

Showing 18 changed files with 405 additions and 140 deletions.
diff --git a/Lib/lib2to3/fixer_base.py b/Lib/lib2to3/fixer_base.py
@@ -33,6 +33,8 @@ class BaseFix(object):
     explicit = False # Is this ignored by refactor.py -f all?
     run_order = 5   # Fixers will be sorted by run order before execution
                     # Lower numbers will be run first.
+    _accept_type = None # [Advanced and not public] This tells RefactoringTool
+                        # which node type to accept when there's not a pattern.
 
     # Shortcut for access to Python grammar symbols
     syms = pygram.python_symbols

diff --git a/Lib/lib2to3/fixes/fix_import.py b/Lib/lib2to3/fixes/fix_import.py
@@ -12,7 +12,7 @@
 
 # Local imports
 from .. import fixer_base
-from os.path import dirname, join, exists, pathsep
+from os.path import dirname, join, exists, sep
 from ..fixer_util import FromImport, syms, token
 
 
@@ -84,7 +84,7 @@ def probably_a_local_import(self, imp_name):
         # so can't be a relative import.
         if not exists(join(dirname(base_path), '__init__.py')):
             return False
-        for ext in ['.py', pathsep, '.pyc', '.so', '.sl', '.pyd']:
+        for ext in ['.py', sep, '.pyc', '.so', '.sl', '.pyd']:
             if exists(base_path + ext):
                 return True
         return False
diff --git a/Lib/lib2to3/fixes/fix_imports.py b/Lib/lib2to3/fixes/fix_imports.py
@@ -84,8 +84,6 @@ def build_pattern(mapping=MAPPING):
 
 class FixImports(fixer_base.BaseFix):
 
-    order = "pre" # Pre-order tree traversal
-
     # This is overridden in fix_imports2.
     mapping = MAPPING
 

diff --git a/Lib/lib2to3/fixes/fix_long.py b/Lib/lib2to3/fixes/fix_long.py
@@ -5,18 +5,15 @@
 """
 
 # Local imports
-from .. import fixer_base
-from ..fixer_util import Name, Number, is_probably_builtin
+from lib2to3 import fixer_base
+from lib2to3.fixer_util import is_probably_builtin
 
 
 class FixLong(fixer_base.BaseFix):
 
     PATTERN = "'long'"
 
-    static_int = Name("int")
-
     def transform(self, node, results):
         if is_probably_builtin(node):
-            new = self.static_int.clone()
-            new.prefix = node.prefix
-            return new
+            node.value = "int"
+            node.changed()
diff --git a/Lib/lib2to3/fixes/fix_ne.py b/Lib/lib2to3/fixes/fix_ne.py
@@ -12,9 +12,11 @@
 class FixNe(fixer_base.BaseFix):
     # This is so simple that we don't need the pattern compiler.
 
+    _accept_type = token.NOTEQUAL
+
     def match(self, node):
         # Override
-        return node.type == token.NOTEQUAL and node.value == "<>"
+        return node.value == "<>"
 
     def transform(self, node, results):
         new = pytree.Leaf(token.NOTEQUAL, "!=", prefix=node.prefix)

diff --git a/Lib/lib2to3/fixes/fix_numliterals.py b/Lib/lib2to3/fixes/fix_numliterals.py
@@ -12,10 +12,11 @@
 class FixNumliterals(fixer_base.BaseFix):
     # This is so simple that we don't need the pattern compiler.
 
+    _accept_type = token.NUMBER
+
     def match(self, node):
         # Override
-        return (node.type == token.NUMBER and
-                (node.value.startswith("0") or node.value[-1] in "Ll"))
+        return (node.value.startswith("0") or node.value[-1] in "Ll")
 
     def transform(self, node, results):
         val = node.value

diff --git a/Lib/lib2to3/fixes/fix_operator.py b/Lib/lib2to3/fixes/fix_operator.py
@@ -0,0 +1,40 @@
+"""Fixer for operator.{isCallable,sequenceIncludes}
+
+operator.isCallable(obj) -> hasattr(obj, '__call__')
+operator.sequenceIncludes(obj) -> operator.contains(obj)
+"""
+
+# Local imports
+from .. import fixer_base
+from ..fixer_util import Call, Name, String
+
+class FixOperator(fixer_base.BaseFix):
+
+    methods = "method=('isCallable'|'sequenceIncludes')"
+    func = "'(' func=any ')'"
+    PATTERN = """
+              power< module='operator'
+                trailer< '.' {methods} > trailer< {func} > >
+              |
+              power< {methods} trailer< {func} > >
+              """.format(methods=methods, func=func)
+
+    def transform(self, node, results):
+        method = results["method"][0]
+
+        if method.value == "sequenceIncludes":
+            if "module" not in results:
+                # operator may not be in scope, so we can't make a change.
+                self.warning(node, "You should use operator.contains here.")
+            else:
+                method.value = "contains"
+                method.changed()
+        elif method.value == "isCallable":
+            if "module" not in results:
+                self.warning(node,
+                             "You should use hasattr(%s, '__call__') here." %
+                             results["func"].value)
+            else:
+                func = results["func"]
+                args = [func.clone(), String(", "), String("'__call__'")]
+                return Call(Name("hasattr"), args, prefix=node.prefix)
diff --git a/Lib/lib2to3/fixes/fix_print.py b/Lib/lib2to3/fixes/fix_print.py
@@ -26,20 +26,15 @@
               )
 
 
-class FixPrint(fixer_base.ConditionalFix):
+class FixPrint(fixer_base.BaseFix):
 
     PATTERN = """
               simple_stmt< any* bare='print' any* > | print_stmt
               """
 
-    skip_on = '__future__.print_function'
-
     def transform(self, node, results):
         assert results
 
-        if self.should_skip(node):
-            return
-
         bare_print = results.get("bare")
 
         if bare_print:

diff --git a/Lib/lib2to3/fixes/fix_urllib.py b/Lib/lib2to3/fixes/fix_urllib.py
@@ -12,13 +12,13 @@
 MAPPING = {'urllib':  [
                 ('urllib.request',
                     ['URLOpener', 'FancyURLOpener', 'urlretrieve',
-                     '_urlopener', 'urlopen', 'urlcleanup']),
+                     '_urlopener', 'urlopen', 'urlcleanup',
+                     'pathname2url', 'url2pathname']),
                 ('urllib.parse',
                     ['quote', 'quote_plus', 'unquote', 'unquote_plus',
-                     'urlencode', 'pathname2url', 'url2pathname', 'splitattr',
-                     'splithost', 'splitnport', 'splitpasswd', 'splitport',
-                     'splitquery', 'splittag', 'splittype', 'splituser',
-                     'splitvalue', ]),
+                     'urlencode', 'splitattr', 'splithost', 'splitnport',
+                     'splitpasswd', 'splitport', 'splitquery', 'splittag',
+                     'splittype', 'splituser', 'splitvalue', ]),
                 ('urllib.error',
                     ['ContentTooShortError'])],
            'urllib2' : [

diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py
@@ -4,19 +4,31 @@
 
 import sys
 import os
+import difflib
 import logging
 import shutil
 import optparse
 
 from . import refactor
 
+
+def diff_texts(a, b, filename):
+    """Return a unified diff of two strings."""
+    a = a.splitlines()
+    b = b.splitlines()
+    return difflib.unified_diff(a, b, filename, filename,
+                                "(original)", "(refactored)",
+                                lineterm="")
+
+
 class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool):
     """
     Prints output to stdout.
     """
 
-    def __init__(self, fixers, options, explicit, nobackups):
+    def __init__(self, fixers, options, explicit, nobackups, show_diffs):
         self.nobackups = nobackups
+        self.show_diffs = show_diffs
         super(StdoutRefactoringTool, self).__init__(fixers, options, explicit)
 
     def log_error(self, msg, *args, **kwargs):
@@ -42,9 +54,17 @@ def write_file(self, new_text, filename, old_text, encoding):
         if not self.nobackups:
             shutil.copymode(backup, filename)
 
-    def print_output(self, lines):
-        for line in lines:
-            print(line)
+    def print_output(self, old, new, filename, equal):
+        if equal:
+            self.log_message("No changes to %s", filename)
+        else:
+            self.log_message("Refactored %s", filename)
+            if self.show_diffs:
+                for line in diff_texts(old, new, filename):
+                    print(line)
+
+def warn(msg):
+    print >> sys.stderr, "WARNING: %s" % (msg,)
 
 
 def main(fixer_pkg, args=None):
@@ -70,9 +90,12 @@ def main(fixer_pkg, args=None):
     parser.add_option("-l", "--list-fixes", action="store_true",
                       help="List available transformations (fixes/fix_*.py)")
     parser.add_option("-p", "--print-function", action="store_true",
-                      help="Modify the grammar so that print() is a function")
+                      help="DEPRECATED Modify the grammar so that print() is "
+                          "a function")
     parser.add_option("-v", "--verbose", action="store_true",
                       help="More verbose logging")
+    parser.add_option("--no-diffs", action="store_true",
+                      help="Don't show diffs of the refactoring")
     parser.add_option("-w", "--write", action="store_true",
                       help="Write back modified files")
     parser.add_option("-n", "--nobackups", action="store_true", default=False,
@@ -81,6 +104,11 @@ def main(fixer_pkg, args=None):
     # Parse command line arguments
     refactor_stdin = False
     options, args = parser.parse_args(args)
+    if not options.write and options.no_diffs:
+        warn("not writing files and not printing diffs; that's not very useful")
+    if options.print_function:
+        warn("-p is deprecated; "
+             "detection of from __future__ import print_function is automatic")
     if not options.write and options.nobackups:
         parser.error("Can't use -n without -w")
     if options.list_fixes:
@@ -104,7 +132,6 @@ def main(fixer_pkg, args=None):
     logging.basicConfig(format='%(name)s: %(message)s', level=level)
 
     # Initialize the refactoring tool
-    rt_opts = {"print_function" : options.print_function}
     avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg))
     unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix)
     explicit = set()
@@ -119,8 +146,8 @@ def main(fixer_pkg, args=None):
     else:
         requested = avail_fixes.union(explicit)
     fixer_names = requested.difference(unwanted_fixes)
-    rt = StdoutRefactoringTool(sorted(fixer_names), rt_opts, sorted(explicit),
-                               options.nobackups)
+    rt = StdoutRefactoringTool(sorted(fixer_names), None, sorted(explicit),
+                               options.nobackups, not options.no_diffs)
 
     # Refactor all files and directories passed as arguments
     if not rt.errors:

diff --git a/Lib/lib2to3/patcomp.py b/Lib/lib2to3/patcomp.py
@@ -14,7 +14,7 @@
 import os
 
 # Fairly local imports
-from .pgen2 import driver, literals, token, tokenize, parse
+from .pgen2 import driver, literals, token, tokenize, parse, grammar
 
 # Really local imports
 from . import pytree
@@ -138,7 +138,7 @@ def compile_basic(self, nodes, repeat=None):
         node = nodes[0]
         if node.type == token.STRING:
             value = str(literals.evalString(node.value))
-            return pytree.LeafPattern(content=value)
+            return pytree.LeafPattern(_type_of_literal(value), value)
         elif node.type == token.NAME:
             value = node.value
             if value.isupper():
@@ -179,6 +179,15 @@ def get_int(self, node):
              "TOKEN": None}
 
 
+def _type_of_literal(value):
+    if value[0].isalpha():
+        return token.NAME
+    elif value in grammar.opmap:
+        return grammar.opmap[value]
+    else:
+        return None
+
+
 def pattern_convert(grammar, raw_node_info):
     """Converts raw node information to a Node or Leaf instance."""
     type, value, context, children = raw_node_info

diff --git a/Lib/lib2to3/pgen2/grammar.py b/Lib/lib2to3/pgen2/grammar.py
@@ -97,6 +97,19 @@ def load(self, filename):
         f.close()
         self.__dict__.update(d)
 
+    def copy(self):
+        """
+        Copy the grammar.
+        """
+        new = self.__class__()
+        for dict_attr in ("symbol2number", "number2symbol", "dfas", "keywords",
+                          "tokens", "symbol2label"):
+            setattr(new, dict_attr, getattr(self, dict_attr).copy())
+        new.labels = self.labels[:]
+        new.states = self.states[:]
+        new.start = self.start
+        return new
+
     def report(self):
         """Dump the grammar tables to standard output, for debugging."""
         from pprint import pprint

diff --git a/Lib/lib2to3/pygram.py b/Lib/lib2to3/pygram.py
@@ -28,4 +28,8 @@ def __init__(self, grammar):
 
 
 python_grammar = driver.load_grammar(_GRAMMAR_FILE)
+
 python_symbols = Symbols(python_grammar)
+
+python_grammar_no_print_statement = python_grammar.copy()
+del python_grammar_no_print_statement.keywords["print"]