Whitespace normalization.

This commit is contained in:
Tim Peters 2003-04-24 16:02:54 +00:00
parent 0822ff7cca
commit 0eadaac7dc
39 changed files with 127 additions and 144 deletions

View file

@ -120,4 +120,3 @@ def search_function(encoding):
# Register the search_function in the Python codec registry
codecs.register(search_function)

View file

@ -19,10 +19,10 @@ def nameprep(label):
continue
newlabel.append(stringprep.map_table_b2(c))
label = u"".join(newlabel)
# Normalize
label = unicodedata.normalize("NFKC", label)
# Prohibit
for c in label:
if stringprep.in_table_c12(c) or \
@ -139,7 +139,7 @@ def ToUnicode(label):
# Step 8: return the result of step 5
return result
### Codec APIs
class Codec(codecs.Codec):
@ -156,7 +156,7 @@ class Codec(codecs.Codec):
return ".".join(result), len(input)
def decode(self,input,errors='strict'):
if errors != 'strict':
raise UnicodeError, "Unsupported error handling "+errors

View file

@ -9,7 +9,7 @@ import codecs
##################### Encoding #####################################
def segregate(str):
"""3.1 Basic code point segregation"""
"""3.1 Basic code point segregation"""
base = []
extended = {}
for c in str:
@ -66,7 +66,7 @@ def insertion_unsort(str, extended):
oldindex = index
delta = 0
oldchar = char
return result
def T(j, bias):
@ -103,7 +103,7 @@ def adapt(delta, first, numchars):
divisions += 36
bias = divisions + (36 * delta // (delta + 38))
return bias
def generate_integers(baselen, deltas):
"""3.4 Bias adaptation"""
@ -155,7 +155,7 @@ def decode_generalized_number(extended, extpos, bias, errors):
return extpos, result
w = w * (36 - t)
j += 1
def insertion_sort(base, extended, errors):
"""3.2 Insertion unsort coding"""
@ -193,7 +193,7 @@ def punycode_decode(text, errors):
base = unicode(base, "ascii", errors)
extended = extended.upper()
return insertion_sort(base, extended, errors)
### Codec APIs
class Codec(codecs.Codec):