mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 10:19:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-96019: Fix caching of decompositions in makeunicodedata (GH-96020)

Browse source
This commit is contained in:
Carl Friedrich Bolz-Tereick 2022-08-19 11:20:44 +02:00 • committed by GitHub
parent ee9f22d346
commit 2d9f252c0c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 1243 additions and 1404 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Misc/NEWS.d/next/Library/2022-08-19-10-19-32.gh-issue-96019.b7uAVP.rst Normal file
View file

@ -0,0 +1,3 @@
Fix a bug in the ``makeunicodedata.py`` script leading to about 13 KiB of
space saving in the ``unicodedata`` module, specifically the character
decomposition data.

2634
Modules/unicodedata_db.h generated
View file

File diff suppressed because it is too large Load diff

10
Tools/unicode/makeunicodedata.py
View file

@ -169,6 +169,7 @@ def makeunicodedata(unicode, trace):
# 2) decomposition data # 2) decomposition data
decomp_data_cache = {}
decomp_data = [0] decomp_data = [0]
decomp_prefix = [""] decomp_prefix = [""]
decomp_index = [0] * len(unicode.chars) decomp_index = [0] * len(unicode.chars)
@ -207,12 +208,15 @@ def makeunicodedata(unicode, trace):
comp_first[l] = 1 comp_first[l] = 1
comp_last[r] = 1 comp_last[r] = 1
comp_pairs.append((l,r,char)) comp_pairs.append((l,r,char))
try: key = tuple(decomp)
i = decomp_data.index(decomp) i = decomp_data_cache.get(key, -1)
except ValueError: if i == -1:
i = len(decomp_data) i = len(decomp_data)
decomp_data.extend(decomp) decomp_data.extend(decomp)
decomp_size = decomp_size + len(decomp) * 2 decomp_size = decomp_size + len(decomp) * 2
decomp_data_cache[key] = i
else:
assert decomp_data[i:i+len(decomp)] == decomp
else: else:
i = 0 i = 0
decomp_index[char] = i decomp_index[char] = i