Fredrik Lundh: here's the 96.6% version of SRE

Author: jeremyhylton

Lib/sre.py

@@ -1,4 +1,3 @@
-# -*- Mode: Python; tab-width: 4 -*-
 #
 # Secret Labs' Regular Expression Engine
 # $Id$
@@ -7,39 +6,127 @@
 #
 # Copyright (c) 1998-2000 by Secret Labs AB.  All rights reserved.
 #
-# This code can only be used for 1.6 alpha testing.  All other use
-# require explicit permission from Secret Labs AB.
-#
 # Portions of this engine have been developed in cooperation with
 # CNRI.  Hewlett-Packard provided funding for 1.6 integration and
 # other compatibility work.
 #
 
-"""
-this is a long string
-"""
-
 import sre_compile
 
+# flags
+I = IGNORECASE = sre_compile.SRE_FLAG_IGNORECASE
+L = LOCALE = sre_compile.SRE_FLAG_LOCALE
+M = MULTILINE = sre_compile.SRE_FLAG_MULTILINE
+S = DOTALL = sre_compile.SRE_FLAG_DOTALL
+X = VERBOSE = sre_compile.SRE_FLAG_VERBOSE
+
 # --------------------------------------------------------------------
 # public interface
 
-def compile(pattern, flags=0):
-    return sre_compile.compile(pattern, _fixflags(flags))
 
 def match(pattern, string, flags=0):
-    return compile(pattern, _fixflags(flags)).match(string)
 
 def search(pattern, string, flags=0):
-    return compile(pattern, _fixflags(flags)).search(string)
+
+def sub(pattern, repl, string, count=0):
+    return _compile(pattern).sub(repl, string, count)
+
+def subn(pattern, repl, string, count=0):
+    return _compile(pattern).subn(repl, string, count)
+
+def split(pattern, string, maxsplit=0):
+    return _compile(pattern).split(string, maxsplit)
 
-# FIXME: etc
+    return _compile(pattern).findall(string, maxsplit)
+
+def compile(pattern, flags=0):
+    return _compile(pattern, flags)
+
+def escape(pattern):
+    s = list(pattern)
+    for i in range(len(pattern)):
+        c = pattern[i]
+        if not ("a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9"):
+            if c == "\000":
+                s[i] = "\\000"
+            else:
+                s[i] = "\\" + c
+    return pattern[:0].join(s)
 
 # --------------------------------------------------------------------
-# helpers
+
+_cache = {}
+_MAXCACHE = 100
+
+def _compile(pattern, flags=0):
+    # internal: compile pattern
+    tp = type(pattern)
+    if tp not in (type(""), type(u"")):
+        return pattern
+    key = (tp, pattern, flags)
+    try:
+        return _cache[key]
+    except KeyError:
+        pass
+    p = sre_compile.compile(pattern, flags)
+    if len(_cache) >= _MAXCACHE:
+        _cache.clear()
+    _cache[key] = p
+    return p
+
+def _sub(pattern, template, string, count=0):
+    # internal: pattern.sub implementation hook
+    return _subn(pattern, template, string, count)[0]
+
+def _expand(match, template):
+    # internal: expand template
+    return template # FIXME
 
-def _fixflags(flags):
-    # convert flag bitmask to sequence
-    assert not flags
-    return ()
+    else:
+        # FIXME: prepare template
+        def filter(match, template=template):
+            return _expand(match, template)
+    n = i = 0
+    s = []
+    append = s.append
+    c = pattern.cursor(string)
+    while not count or n < count:
+        m = c.search()
+        if not m:
+            break
+        j = m.start()
+        if j > i:
+            append(string[i:j])
+        append(filter(m))
+        i = m.end()
+        n = n + 1
+    if i < len(string):
+        append(string[i:])
+    return string[:0].join(s), n
 
+def _split(pattern, string, maxsplit=0):
+    # internal: pattern.split implementation hook
+    n = i = 0
+    s = []
+    append = s.append
+    c = pattern.cursor(string)
+    while not maxsplit or n < maxsplit:
+        m = c.search()
+        if not m:
+            break
+        j = m.start()
+        append(string[i:j])
+        i = m.end()
+        n = n + 1
+    if i < len(string):
+        append(string[i:])
+    return s

Lib/sre_compile.py

@@ -14,9 +14,6 @@
 # other compatibility work.
 #
 
-# FIXME: <fl> formalize (objectify?) and document the compiler code
-# format, so that other frontends can use this compiler
-
 import array, string, sys
 
 import _sre
@@ -45,64 +42,70 @@ def append(self, code):
 	self.data.append(code)
     def todata(self):
 	# print self.data
-	return array.array(WORDSIZE, self.data).tostring()
-
-def _lower(literal):
-    # return _sre._lower(literal) # FIXME
-    return string.lower(literal)
 
-def _compile(code, pattern, flags):
     append = code.append
     for op, av in pattern:
 	if op is ANY:
-	    if "s" in flags:
-		append(CODES[op]) # any character at all!
 	    else:
-		append(CODES[NOT_LITERAL])
-		append(10)
 	elif op in (SUCCESS, FAILURE):
-	    append(CODES[op])
 	elif op is AT:
-	    append(CODES[op])
-	    append(POSITIONS[av])
+		append(ATCODES[AT_MULTILINE[av]])
+	    else:
+		append(ATCODES[av])
 	elif op is BRANCH:
-	    append(CODES[op])
 	    tail = []
 	    for av in av[1]:
 		skip = len(code); append(0)
-		_compile(code, av, flags)
-		append(CODES[JUMP])
 		tail.append(len(code)); append(0)
 		code[skip] = len(code) - skip
 	    append(0) # end of branch
 	    for tail in tail:
 		code[tail] = len(code) - tail
 	elif op is CALL:
-	    append(CODES[op])
 	    skip = len(code); append(0)
-	    _compile(code, av, flags)
-	    append(CODES[SUCCESS])
 	    code[skip] = len(code) - skip
 	elif op is CATEGORY: # not used by current parser
-	    append(CODES[op])
-	    append(CATEGORIES[av])
+		append(CH_LOCALE[CHCODES[av]])
+	    else:
+		append(CHCODES[av])
 	elif op is GROUP:
-	    if "i" in flags:
-		append(CODES[MAP_IGNORE[op]])
 	    else:
-		append(CODES[op])
-	    append(av)
 	elif op is IN:
-	    if "i" in flags:
-		append(CODES[MAP_IGNORE[op]])
 		def fixup(literal):
-		    return ord(_lower(literal))
 	    else:
-		append(CODES[op])
 		fixup = ord
 	    skip = len(code); append(0)
 	    for op, av in av:
-		append(CODES[op])
 		if op is NEGATE:
 		    pass
 		elif op is LITERAL:
@@ -111,77 +114,80 @@ def fixup(literal):
 		    append(fixup(av[0]))
 		    append(fixup(av[1]))
 		elif op is CATEGORY:
-		    append(CATEGORIES[av])
+			append(CH_LOCALE[CHCODES[av]])
+		    else:
+			append(CHCODES[av])
 		else:
 		    raise ValueError, "unsupported set operator"
-	    append(CODES[FAILURE])
 	    code[skip] = len(code) - skip
 	elif op in (LITERAL, NOT_LITERAL):
-	    if "i" in flags:
-		append(CODES[MAP_IGNORE[op]])
-		append(ord(_lower(av)))
 	    else:
-		append(CODES[op])
 		append(ord(av))
 	elif op is MARK:
-	    append(CODES[op])
 	    append(av)
  	elif op in (REPEAT, MIN_REPEAT, MAX_REPEAT):
 	    lo, hi = av[2].getwidth()
  	    if lo == 0:
  		raise SyntaxError, "cannot repeat zero-width items"
 	    if lo == hi == 1 and op is MAX_REPEAT:
-		append(CODES[MAX_REPEAT_ONE])
 		skip = len(code); append(0)
 		append(av[0])
 		append(av[1])
-		_compile(code, av[2], flags)
-		append(CODES[SUCCESS])
 		code[skip] = len(code) - skip
 	    else:
-		append(CODES[op])
 		skip = len(code); append(0)
 		append(av[0])
 		append(av[1])
-		_compile(code, av[2], flags)
 		if op is MIN_REPEAT:
-		    append(CODES[MIN_UNTIL])
 		else:
-		    # FIXME: MAX_REPEAT PROBABLY DOESN'T WORK (?)
-		    append(CODES[MAX_UNTIL])
 		code[skip] = len(code) - skip
 	elif op is SUBPATTERN:
-## 	    group = av[0]
-## 	    if group:
-## 		append(CODES[MARK])
-## 		append((group-1)*2)
-	    _compile(code, av[1], flags)
-## 	    if group:
-## 		append(CODES[MARK])
-## 		append((group-1)*2+1)
 	else:
 	    raise ValueError, ("unsupported operand type", op)
 
-def compile(p, flags=()):
     # convert pattern list to internal format
     if type(p) in (type(""), type(u"")):
 	import sre_parse
 	pattern = p
 	p = sre_parse.parse(p)
     else:
 	pattern = None
-    # print p.getwidth()
-    # print p
     code = Code()
-    _compile(code, p.data, p.pattern.flags)
-    code.append(CODES[SUCCESS])
-    # print list(code.data)
     data = code.todata()
     if 0: # debugging
 	print
 	print "-" * 68
 	import sre_disasm
 	sre_disasm.disasm(data)
 	print "-" * 68
-    # print len(data), p.pattern.groups, len(p.pattern.groupdict)
-    return _sre.compile(pattern, data, p.pattern.groups-1, p.pattern.groupdict)
+	data,
+	p.pattern.groups-1, p.pattern.groupdict
+	)