From 01b1c7b23860ccf224ee17ca6f2880f6a3fc81e8 Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Sat, 24 Sep 2022 10:14:24 -0300 Subject: [PATCH] Show all the items in tuples and lists. Fixes #1056 --- .../pydevd/_pydevd_bundle/pydevd_constants.py | 19 ++ .../pydevd/_pydevd_bundle/pydevd_resolver.py | 164 +++++++++++++++--- .../pydevd/_pydevd_bundle/pydevd_vars.py | 30 ++-- .../pydevd/_pydevd_bundle/pydevd_xml.py | 4 +- .../types/pydevd_plugin_numpy_types.py | 12 +- .../pydevd/tests_python/debugger_unittest.py | 42 ++++- .../pydevd/tests_python/test_debugger.py | 99 ++++++++++- .../test_case_variable_access.json | 19 ++ .../test_case_variable_access_by_id.json | 8 + .../pydevd/tests_python/test_resolvers.py | 137 +++++++++++++++ ...variables_dict_multiple_levels_dap_2_.json | 10 ++ ...riables_dict_multiple_levels_dap_300_.json | 15 ++ ...hild_variables_multiple_levels_dap_2_.json | 28 +++ ...ld_variables_multiple_levels_dap_300_.json | 23 +++ ...variables_multiple_levels_resolver_2_.json | 25 +++ ...riables_multiple_levels_resolver_300_.json | 19 ++ .../test_suspended_frames_manager.py | 18 +- 17 files changed, 609 insertions(+), 63 deletions(-) create mode 100644 src/debugpy/_vendored/pydevd/tests_python/test_debugger/test_case_variable_access.json create mode 100644 src/debugpy/_vendored/pydevd/tests_python/test_debugger/test_case_variable_access_by_id.json create mode 100644 src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_dict_multiple_levels_dap_2_.json create mode 100644 src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_dict_multiple_levels_dap_300_.json create mode 100644 src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_dap_2_.json create mode 100644 src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_dap_300_.json create mode 100644 src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_resolver_2_.json create mode 100644 src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_resolver_300_.json diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py index 884d85fe..a9bd9308 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_constants.py @@ -266,6 +266,25 @@ else: # If not specified, uses default heuristic to determine if it should be loaded. PYDEVD_USE_FRAME_EVAL = os.getenv('PYDEVD_USE_FRAME_EVAL', '').lower() +# Values used to determine how much container items will be shown. +# PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS: +# - Defines how many items will appear initially expanded after which a 'more...' will appear. +# +# PYDEVD_CONTAINER_BUCKET_SIZE +# - Defines the size of each bucket inside the 'more...' item +# i.e.: a bucket with size == 2 would show items such as: +# - [2:4] +# - [4:6] +# ... +# +# PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS +# - Defines the maximum number of items for dicts and sets. +# +PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS = as_int_in_env('PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS', 100) +PYDEVD_CONTAINER_BUCKET_SIZE = as_int_in_env('PYDEVD_CONTAINER_BUCKET_SIZE', 1000) +PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS = as_int_in_env('PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS', 500) +PYDEVD_CONTAINER_NUMPY_MAX_ITEMS = as_int_in_env('PYDEVD_CONTAINER_NUMPY_MAX_ITEMS', 500) + PYDEVD_IPYTHON_COMPATIBLE_DEBUGGING = is_true_in_env('PYDEVD_IPYTHON_COMPATIBLE_DEBUGGING') # If specified in PYDEVD_IPYTHON_CONTEXT it must be a string with the basename diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_resolver.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_resolver.py index ac7aa100..9d97179b 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_resolver.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_resolver.py @@ -8,12 +8,9 @@ from functools import partial from _pydevd_bundle.pydevd_constants import IS_PY36_OR_GREATER, \ MethodWrapperType, RETURN_VALUES_DICT, DebugInfoHolder, IS_PYPY, GENERATED_LEN_ATTR_NAME from _pydevd_bundle.pydevd_safe_repr import SafeRepr +from _pydevd_bundle import pydevd_constants -# Note: 300 is already a lot to see in the outline (after that the user should really use the shell to get things) -# and this also means we'll pass less information to the client side (which makes debugging faster). -MAX_ITEMS_TO_HANDLE = 300 - -TOO_LARGE_MSG = 'Too large to show contents. Max items to show: ' + str(MAX_ITEMS_TO_HANDLE) +TOO_LARGE_MSG = 'Maximum number of items (%s) reached. To show more items customize the value of the PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS environment variable.' TOO_LARGE_ATTR = 'Unable to handle:' @@ -311,8 +308,8 @@ class DictResolver: else: eval_key_str = None ret.append((key_as_str, val, eval_key_str)) - if i > MAX_ITEMS_TO_HANDLE: - ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG, None)) + if i >= pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS: + ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG % (pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS,), None)) break # in case the class extends built-in type and has some additional fields @@ -336,8 +333,8 @@ class DictResolver: # we need to add the id because otherwise we cannot find the real object to get its contents later on. key = '%s (%s)' % (self.key_to_str(key), id(key)) ret[key] = val - if i > MAX_ITEMS_TO_HANDLE: - ret[TOO_LARGE_ATTR] = TOO_LARGE_MSG + if i >= pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS: + ret[TOO_LARGE_ATTR] = TOO_LARGE_MSG % (pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS,) break # in case if the class extends built-in type and has some additional fields @@ -351,21 +348,127 @@ def _apply_evaluate_name(parent_name, evaluate_name): return evaluate_name % (parent_name,) -#======================================================================================================================= -# TupleResolver -#======================================================================================================================= +class MoreItemsRange: + + def __init__(self, value, from_i, to_i): + self.value = value + self.from_i = from_i + self.to_i = to_i + + def get_contents_debug_adapter_protocol(self, _self, fmt=None): + l = len(self.value) + ret = [] + + format_str = '%0' + str(int(len(str(l - 1)))) + 'd' + if fmt is not None and fmt.get('hex', False): + format_str = '0x%0' + str(int(len(hex(l).lstrip('0x')))) + 'x' + + for i, item in enumerate(self.value[self.from_i:self.to_i]): + i += self.from_i + ret.append((format_str % i, item, '[%s]' % i)) + return ret + + def get_dictionary(self, _self, fmt=None): + dct = {} + for key, obj, _ in self.get_contents_debug_adapter_protocol(self, fmt): + dct[key] = obj + return dct + + def resolve(self, attribute): + ''' + :param var: that's the original object we're dealing with. + :param attribute: that's the key to resolve + -- either the dict key in get_dictionary or the name in the dap protocol. + ''' + return self.value[int(attribute)] + + def __eq__(self, o): + return isinstance(o, MoreItemsRange) and self.value is o.value and \ + self.from_i == o.from_i and self.to_i == o.to_i + + def __str__(self): + return '[%s:%s]' % (self.from_i, self.to_i) + + __repr__ = __str__ + + +class MoreItems: + + def __init__(self, value, handled_items): + self.value = value + self.handled_items = handled_items + + def get_contents_debug_adapter_protocol(self, _self, fmt=None): + total_items = len(self.value) + remaining = total_items - self.handled_items + bucket_size = pydevd_constants.PYDEVD_CONTAINER_BUCKET_SIZE + + from_i = self.handled_items + to_i = from_i + min(bucket_size, remaining) + + ret = [] + while remaining > 0: + remaining -= bucket_size + more_items_range = MoreItemsRange(self.value, from_i, to_i) + ret.append((str(more_items_range), more_items_range, None)) + + from_i = to_i + to_i = from_i + min(bucket_size, remaining) + + return ret + + def get_dictionary(self, _self, fmt=None): + dct = {} + for key, obj, _ in self.get_contents_debug_adapter_protocol(self, fmt): + dct[key] = obj + return dct + + def resolve(self, attribute): + from_i, to_i = attribute[1:-1].split(':') + from_i = int(from_i) + to_i = int(to_i) + return MoreItemsRange(self.value, from_i, to_i) + + def __eq__(self, o): + return isinstance(o, MoreItems) and self.value is o.value + + def __str__(self): + return '...' + + __repr__ = __str__ + + +class ForwardInternalResolverToObject: + ''' + To be used when we provide some internal object that'll actually do the resolution. + ''' + + def get_contents_debug_adapter_protocol(self, obj, fmt=None): + return obj.get_contents_debug_adapter_protocol(fmt) + + def get_dictionary(self, var, fmt={}): + return var.get_dictionary(var, fmt) + + def resolve(self, var, attribute): + return var.resolve(attribute) + + class TupleResolver: # to enumerate tuples and lists def resolve(self, var, attribute): ''' - @param var: that's the original attribute - @param attribute: that's the key passed in the dict (as a string) + :param var: that's the original object we're dealing with. + :param attribute: that's the key to resolve + -- either the dict key in get_dictionary or the name in the dap protocol. ''' if attribute in (GENERATED_LEN_ATTR_NAME, TOO_LARGE_ATTR): return None try: return var[int(attribute)] except: + if attribute == 'more': + return MoreItems(var, pydevd_constants.PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS) + return getattr(var, attribute) def get_contents_debug_adapter_protocol(self, lst, fmt=None): @@ -378,18 +481,26 @@ class TupleResolver: # to enumerate tuples and lists :return list(tuple(name:str, value:object, evaluateName:str)) ''' - l = len(lst) + lst_len = len(lst) ret = [] - format_str = '%0' + str(int(len(str(l - 1)))) + 'd' + format_str = '%0' + str(int(len(str(lst_len - 1)))) + 'd' if fmt is not None and fmt.get('hex', False): - format_str = '0x%0' + str(int(len(hex(l).lstrip('0x')))) + 'x' + format_str = '0x%0' + str(int(len(hex(lst_len).lstrip('0x')))) + 'x' + initial_expanded = pydevd_constants.PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS for i, item in enumerate(lst): ret.append((format_str % i, item, '[%s]' % i)) - if i > MAX_ITEMS_TO_HANDLE: - ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG, None)) + if i >= initial_expanded - 1: + if (lst_len - initial_expanded) < pydevd_constants.PYDEVD_CONTAINER_BUCKET_SIZE: + # Special case: if we have just 1 more bucket just put it inline. + item = MoreItemsRange(lst, initial_expanded, lst_len) + + else: + # Multiple buckets + item = MoreItems(lst, initial_expanded) + ret.append(('more', item, None)) break # Needed in case the class extends the built-in type and has some additional fields. @@ -408,11 +519,13 @@ class TupleResolver: # to enumerate tuples and lists if fmt is not None and fmt.get('hex', False): format_str = '0x%0' + str(int(len(hex(l).lstrip('0x')))) + 'x' + initial_expanded = pydevd_constants.PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS for i, item in enumerate(var): d[format_str % i] = item - if i > MAX_ITEMS_TO_HANDLE: - d[TOO_LARGE_ATTR] = TOO_LARGE_MSG + if i >= initial_expanded - 1: + item = MoreItems(var, initial_expanded) + d['more'] = item break # in case if the class extends built-in type and has some additional fields @@ -436,8 +549,8 @@ class SetResolver: for i, item in enumerate(obj): ret.append((str(id(item)), item, None)) - if i > MAX_ITEMS_TO_HANDLE: - ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG, None)) + if i >= pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS: + ret.append((TOO_LARGE_ATTR, TOO_LARGE_MSG % (pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS,), None)) break # Needed in case the class extends the built-in type and has some additional fields. @@ -467,8 +580,8 @@ class SetResolver: for i, item in enumerate(var): d[str(id(item))] = item - if i > MAX_ITEMS_TO_HANDLE: - d[TOO_LARGE_ATTR] = TOO_LARGE_MSG + if i >= pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS: + d[TOO_LARGE_ATTR] = TOO_LARGE_MSG % (pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS,) break # in case if the class extends built-in type and has some additional fields @@ -670,6 +783,7 @@ dequeResolver = DequeResolver() orderedDictResolver = OrderedDictResolver() frameResolver = FrameResolver() dapGrouperResolver = DAPGrouperResolver() +forwardInternalResolverToObject = ForwardInternalResolverToObject() class InspectStub: diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_vars.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_vars.py index 80100938..0744abf2 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_vars.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_vars.py @@ -47,7 +47,7 @@ def dump_frames(thread_id): @silence_warnings_decorator -def getVariable(dbg, thread_id, frame_id, scope, attrs): +def getVariable(dbg, thread_id, frame_id, scope, locator): """ returns the value of a variable @@ -55,7 +55,7 @@ def getVariable(dbg, thread_id, frame_id, scope, attrs): BY_ID means we'll traverse the list of all objects alive to get the object. - :attrs: after reaching the proper scope, we have to get the attributes until we find + :locator: after reaching the proper scope, we have to get the attributes until we find the proper location (i.e.: obj\tattr1\tattr2) :note: when BY_ID is used, the frame_id is considered the id of the object to find and @@ -74,15 +74,15 @@ def getVariable(dbg, thread_id, frame_id, scope, attrs): frame_id = int(frame_id) for var in objects: if id(var) == frame_id: - if attrs is not None: - attrList = attrs.split('\t') - for k in attrList: + if locator is not None: + locator_parts = locator.split('\t') + for k in locator_parts: _type, _type_name, resolver = get_type(var) var = resolver.resolve(var, k) return var - # If it didn't return previously, we coudn't find it by id (i.e.: alrceady garbage collected). + # If it didn't return previously, we coudn't find it by id (i.e.: already garbage collected). sys.stderr.write('Unable to find object with id: %s\n' % (frame_id,)) return None @@ -90,33 +90,33 @@ def getVariable(dbg, thread_id, frame_id, scope, attrs): if frame is None: return {} - if attrs is not None: - attrList = attrs.split('\t') + if locator is not None: + locator_parts = locator.split('\t') else: - attrList = [] + locator_parts = [] - for attr in attrList: + for attr in locator_parts: attr.replace("@_@TAB_CHAR@_@", '\t') if scope == 'EXPRESSION': - for count in range(len(attrList)): + for count in range(len(locator_parts)): if count == 0: # An Expression can be in any scope (globals/locals), therefore it needs to evaluated as an expression - var = evaluate_expression(dbg, frame, attrList[count], False) + var = evaluate_expression(dbg, frame, locator_parts[count], False) else: _type, _type_name, resolver = get_type(var) - var = resolver.resolve(var, attrList[count]) + var = resolver.resolve(var, locator_parts[count]) else: if scope == "GLOBAL": var = frame.f_globals - del attrList[0] # globals are special, and they get a single dummy unused attribute + del locator_parts[0] # globals are special, and they get a single dummy unused attribute else: # in a frame access both locals and globals as Python does var = {} var.update(frame.f_globals) var.update(frame.f_locals) - for k in attrList: + for k in locator_parts: _type, _type_name, resolver = get_type(var) var = resolver.resolve(var, k) diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_xml.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_xml.py index b523ffaf..c235086b 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_xml.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_xml.py @@ -7,7 +7,7 @@ from _pydevd_bundle.pydevd_constants import BUILTINS_MODULE_NAME, MAXIMUM_VARIAB from _pydev_bundle.pydev_imports import quote from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider, StrPresentationProvider from _pydevd_bundle.pydevd_utils import isinstance_checked, hasattr_checked, DAPGrouper -from _pydevd_bundle.pydevd_resolver import get_var_scope +from _pydevd_bundle.pydevd_resolver import get_var_scope, MoreItems, MoreItemsRange try: import types @@ -60,6 +60,8 @@ def _create_default_type_map(): pass # not available on all python versions default_type_map.append((DAPGrouper, pydevd_resolver.dapGrouperResolver)) + default_type_map.append((MoreItems, pydevd_resolver.forwardInternalResolverToObject)) + default_type_map.append((MoreItemsRange, pydevd_resolver.forwardInternalResolverToObject)) try: default_type_map.append((set, pydevd_resolver.setResolver)) diff --git a/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/types/pydevd_plugin_numpy_types.py b/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/types/pydevd_plugin_numpy_types.py index 1e3e6e90..571f7a9c 100644 --- a/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/types/pydevd_plugin_numpy_types.py +++ b/src/debugpy/_vendored/pydevd/pydevd_plugins/extensions/types/pydevd_plugin_numpy_types.py @@ -1,6 +1,10 @@ from _pydevd_bundle.pydevd_extension_api import TypeResolveProvider -from _pydevd_bundle.pydevd_resolver import defaultResolver, MAX_ITEMS_TO_HANDLE, TOO_LARGE_ATTR, TOO_LARGE_MSG +from _pydevd_bundle.pydevd_resolver import defaultResolver from .pydevd_helpers import find_mod_attr +from _pydevd_bundle import pydevd_constants + +TOO_LARGE_MSG = 'Maximum number of items (%s) reached. To show more items customize the value of the PYDEVD_CONTAINER_NUMPY_MAX_ITEMS environment variable.' +TOO_LARGE_ATTR = 'Unable to handle:' class NdArrayItemsContainer(object): @@ -47,8 +51,8 @@ class NDArrayTypeResolveProvider(object): for item in obj: setattr(container, format_str % i, item) i += 1 - if i > MAX_ITEMS_TO_HANDLE: - setattr(container, TOO_LARGE_ATTR, TOO_LARGE_MSG) + if i >= pydevd_constants.PYDEVD_CONTAINER_NUMPY_MAX_ITEMS: + setattr(container, TOO_LARGE_ATTR, TOO_LARGE_MSG % (pydevd_constants.PYDEVD_CONTAINER_NUMPY_MAX_ITEMS,)) break return container return None @@ -73,7 +77,7 @@ class NDArrayTypeResolveProvider(object): ret['dtype'] = obj.dtype ret['size'] = obj.size try: - ret['[0:%s] ' % (len(obj))] = list(obj[0:MAX_ITEMS_TO_HANDLE]) + ret['[0:%s] ' % (len(obj))] = list(obj[0:pydevd_constants.PYDEVD_CONTAINER_NUMPY_MAX_ITEMS]) except: # This may not work depending on the array shape. pass diff --git a/src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py b/src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py index bdf7008c..f2dadf8c 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py +++ b/src/debugpy/_vendored/pydevd/tests_python/debugger_unittest.py @@ -1330,7 +1330,7 @@ class AbstractWriterThread(threading.Thread): traceback.print_exc() raise AssertionError('Unable to parse:\n%s\njson:\n%s' % (last, json_msg)) - def wait_for_message(self, accept_message, unquote_msg=True, expect_xml=True, timeout=None): + def wait_for_message(self, accept_message, unquote_msg=True, expect_xml=True, timeout=None, double_unquote=True): if isinstance(accept_message, (str, int)): msg_starts_with = '%s\t' % (accept_message,) @@ -1343,7 +1343,12 @@ class AbstractWriterThread(threading.Thread): while True: last = self.get_next_message('wait_for_message', timeout=timeout) if unquote_msg: - last = unquote_plus(unquote_plus(last)) + last = unquote_plus(last) + if double_unquote: + # This is useful if the checking will be done without needing to unpack the + # actual xml (in which case we'll be unquoting things inside of attrs -- + # this could actually make the xml invalid though). + last = unquote_plus(last) if accept_message(last): if expect_xml: # Extract xml and return untangled. @@ -1366,6 +1371,39 @@ class AbstractWriterThread(threading.Thread): prev = last + def wait_for_untangled_message(self, accept_message, timeout=None, double_unquote=False): + import untangle + from io import StringIO + prev = None + while True: + last = self.get_next_message('wait_for_message', timeout=timeout) + last = unquote_plus(last) + if double_unquote: + last = unquote_plus(last) + # Extract xml with untangled. + xml = '' + try: + xml = last[last.index(''):] + except: + traceback.print_exc() + raise AssertionError('Unable to find xml in: %s' % (last,)) + + try: + if isinstance(xml, bytes): + xml = xml.decode('utf-8') + xml = untangle.parse(StringIO(xml)) + except: + traceback.print_exc() + raise AssertionError('Unable to parse:\n%s\nxml:\n%s' % (last, xml)) + untangled = xml.xml + cmd_id = last.split('\t', 1)[0] + if accept_message(int(cmd_id), untangled): + return untangled + if prev != last: + print('Ignored message: %r' % (last,)) + + prev = last + def get_frame_names(self, thread_id): self.write_get_thread_stack(thread_id) msg = self.wait_for_message(CMD_GET_THREAD_STACK) diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py index 7057252a..5aabae18 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py +++ b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py @@ -18,16 +18,17 @@ from tests_python.debugger_unittest import (CMD_SET_PROPERTY_TRACE, REASON_CAUGH CMD_THREAD_SUSPEND, CMD_STEP_OVER, REASON_STEP_OVER, CMD_THREAD_SUSPEND_SINGLE_NOTIFICATION, CMD_THREAD_RESUME_SINGLE_NOTIFICATION, REASON_STEP_RETURN, REASON_STEP_RETURN_MY_CODE, REASON_STEP_OVER_MY_CODE, REASON_STEP_INTO, CMD_THREAD_KILL, IS_PYPY, REASON_STOP_ON_START, - CMD_SMART_STEP_INTO) + CMD_SMART_STEP_INTO, CMD_GET_VARIABLE) from _pydevd_bundle.pydevd_constants import IS_WINDOWS, IS_PY38_OR_GREATER, \ IS_MAC -from _pydevd_bundle.pydevd_comm_constants import CMD_RELOAD_CODE, CMD_INPUT_REQUESTED +from _pydevd_bundle.pydevd_comm_constants import CMD_RELOAD_CODE, CMD_INPUT_REQUESTED, \ + CMD_RUN_CUSTOM_OPERATION import json import pydevd_file_utils import subprocess import threading from _pydev_bundle import pydev_log -from urllib.parse import unquote +from urllib.parse import unquote, unquote_plus from tests_python.debug_constants import * # noqa @@ -1732,6 +1733,98 @@ def test_case_type_ext(case_setup): writer.finished_ok = True +def test_case_variable_access(case_setup, pyfile, data_regression): + + @pyfile + def case_custom(): + obj = [ + tuple(range(9)), + [ + tuple(range(5)), + ] + ] + + print('TEST SUCEEDED') + + with case_setup.test_file(case_custom) as writer: + line = writer.get_line_index_with_content('TEST SUCEEDED') + writer.write_add_breakpoint(line) + writer.write_make_initial_run() + + hit = writer.wait_for_breakpoint_hit('111') + writer.write_get_frame(hit.thread_id, hit.frame_id) + + frame_vars = writer.wait_for_untangled_message( + accept_message=lambda cmd_id, untangled: cmd_id == CMD_GET_FRAME) + + obj_var = [v for v in frame_vars.var if v['name'] == 'obj'][0] + assert obj_var['type'] == 'list' + assert unquote_plus(obj_var['value']) == ": [(0, 1, 2, 3, 4, 5, 6, 7, 8), [(0, 1, 2, 3, 4)]]" + assert obj_var['isContainer'] == "True" + + def _skip_key_in_dict(key): + try: + int(key) + except ValueError: + if 'more' in key or '[' in key: + return False + return True + return False + + def collect_vars(locator, level=0): + writer.write("%s\t%s\t%s\t%s" % (CMD_GET_VARIABLE, writer.next_seq(), hit.thread_id, locator)) + obj_vars = writer.wait_for_untangled_message( + accept_message=lambda cmd_id, _untangled: cmd_id == CMD_GET_VARIABLE) + + for v in obj_vars.var: + if _skip_key_in_dict(v['name']): + continue + new_locator = locator + '\t' + v['name'] + yield level, v, new_locator + if v['isContainer'] == 'True': + yield from collect_vars(new_locator, level + 1) + + found = [] + for level, val, _locator in collect_vars('%s\tFRAME\tobj' % hit.frame_id): + found.append(((' ' * level) + val['name'] + ': ' + unquote_plus(val['value']))) + + data_regression.check(found) + + # Check referrers + full_loc = '%s\t%s\t%s' % (hit.thread_id, hit.frame_id, 'FRAME\tobj\t1\t0') + writer.write_custom_operation(full_loc, 'EXEC', "from _pydevd_bundle.pydevd_referrers import get_referrer_info", "get_referrer_info") + msg = writer.wait_for_untangled_message( + double_unquote=True, + accept_message=lambda cmd_id, _untangled: cmd_id == CMD_RUN_CUSTOM_OPERATION) + + msg_vars = msg.var + try: + msg_vars['found_as'] + msg_vars = [msg_vars] + except: + pass # it's a container. + + for v in msg_vars: + if v['found_as'] == 'list[0]': + # In pypy we may have more than one reference, find out the one + referrer_id = v['id'] + assert int(referrer_id) + assert unquote_plus(v['value']) == ": [(0, 1, 2, 3, 4)]" + break + else: + raise AssertionError("Unable to find ref with list[0]. Found: %s" % (msg_vars,)) + + found = [] + by_id_locator = '%s\t%s' % (referrer_id, 'BY_ID') + for level, val, _locator in collect_vars(by_id_locator): + found.append(((' ' * level) + val['name'] + ': ' + unquote_plus(val['value']))) + + data_regression.check(found, basename='test_case_variable_access_by_id') + + writer.write_run_thread(hit.thread_id) + writer.finished_ok = True + + @pytest.mark.skipif(IS_IRONPYTHON or IS_JYTHON, reason='Failing on IronPython and Jython (needs to be investigated).') def test_case_event_ext(case_setup): diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_debugger/test_case_variable_access.json b/src/debugpy/_vendored/pydevd/tests_python/test_debugger/test_case_variable_access.json new file mode 100644 index 00000000..69205050 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/tests_python/test_debugger/test_case_variable_access.json @@ -0,0 +1,19 @@ +[ + "0: : (0, 1, 2, 3, 4, 5, 6, 7, 8)", + " 0: int: 0", + " 1: int: 1", + " 2: int: 2", + " 3: int: 3", + " 4: int: 4", + " 5: int: 5", + " 6: int: 6", + " 7: int: 7", + " 8: int: 8", + "1: : [(0, 1, 2, 3, 4)]", + " 0: : (0, 1, 2, 3, 4)", + " 0: int: 0", + " 1: int: 1", + " 2: int: 2", + " 3: int: 3", + " 4: int: 4" +] \ No newline at end of file diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_debugger/test_case_variable_access_by_id.json b/src/debugpy/_vendored/pydevd/tests_python/test_debugger/test_case_variable_access_by_id.json new file mode 100644 index 00000000..51a07587 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/tests_python/test_debugger/test_case_variable_access_by_id.json @@ -0,0 +1,8 @@ +[ + "0: : (0, 1, 2, 3, 4)", + " 0: int: 0", + " 1: int: 1", + " 2: int: 2", + " 3: int: 3", + " 4: int: 4" +] \ No newline at end of file diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_resolvers.py b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers.py index e7710c8a..1e6b4134 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/test_resolvers.py +++ b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers.py @@ -1,4 +1,7 @@ from _pydevd_bundle.pydevd_constants import IS_PY36_OR_GREATER, GENERATED_LEN_ATTR_NAME +from _pydevd_bundle import pydevd_constants, pydevd_frame_utils +import pytest +import sys def check_len_entry(len_entry, first_2_params): @@ -360,3 +363,137 @@ def test_tuple_resolver_ctypes(): '2': 3, GENERATED_LEN_ATTR_NAME: 3 } + + +def get_tuple_recursive(): + obj = [ + tuple(range(9)), + [ + tuple(range(5)), + ] + ] + + return sys._getframe() + + +def get_dict_recursive(): + obj = { + 1:1, + 2:{ + 3:3, + 4:4, + 5:5, + 6:6, + 7:{ + 8:8, + 9:9, + 10:10} + } + } + + return sys._getframe() + + +class _DummyPyDB(object): + + def __init__(self): + from _pydevd_bundle.pydevd_api import PyDevdAPI + self.variable_presentation = PyDevdAPI.VariablePresentation() + + +class _DAPCheckChildVars: + + def __init__(self, data_regression, monkeypatch): + self.data_regression = data_regression + self.monkeypatch = monkeypatch + + def check(self, frame, initial_expanded): + self.monkeypatch.setattr(pydevd_constants, 'PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS', initial_expanded + 2) + self.monkeypatch.setattr(pydevd_constants, 'PYDEVD_CONTAINER_BUCKET_SIZE', initial_expanded) + + from _pydevd_bundle.pydevd_suspended_frames import SuspendedFramesManager + suspended_frames_manager = SuspendedFramesManager() + py_db = _DummyPyDB() + + # Now, let's enable the list packing with less items. + with suspended_frames_manager.track_frames(py_db) as tracker: + # : :type tracker: _FramesTracker + thread_id = 'thread1' + tracker.track(thread_id, pydevd_frame_utils.create_frames_list_from_frame(frame)) + + assert suspended_frames_manager.get_thread_id_for_variable_reference(id(frame)) == thread_id + + found = [] + frame_var = suspended_frames_manager.get_variable(id(frame)) + for level, variable in self.collect_all_dap(frame_var.get_child_variable_named('obj')): + found.append(((' ' * level) + variable.name + ': ' + str(variable.value))) + + self.data_regression.check(found) + + def collect_all_dap(self, variable, level=0): + children_variables = variable.get_children_variables() + for var in children_variables: + if var.name in ('special variables', 'function variables'): + continue + yield level, var + yield from self.collect_all_dap(var, level + 1) + + +@pytest.fixture +def dap_check_child_vars(data_regression, monkeypatch): + yield _DAPCheckChildVars(data_regression, monkeypatch) + + +@pytest.mark.parametrize('initial_expanded', [300, 2]) +def test_get_child_variables_multiple_levels_dap(initial_expanded, dap_check_child_vars): + frame = get_tuple_recursive() + dap_check_child_vars.check(frame, initial_expanded) + + +@pytest.mark.parametrize('initial_expanded', [300, 2]) +def test_get_child_variables_dict_multiple_levels_dap(initial_expanded, dap_check_child_vars, monkeypatch): + monkeypatch.setattr(pydevd_constants, 'PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS', initial_expanded) + frame = get_dict_recursive() + dap_check_child_vars.check(frame, initial_expanded) + + +@pytest.mark.parametrize('initial_expanded', [300, 2]) +def test_get_child_variables_multiple_levels_resolver(data_regression, initial_expanded, monkeypatch): + monkeypatch.setattr(pydevd_constants, 'PYDEVD_CONTAINER_INITIAL_EXPANDED_ITEMS', initial_expanded + 2) + monkeypatch.setattr(pydevd_constants, 'PYDEVD_CONTAINER_BUCKET_SIZE', initial_expanded) + + obj = [ + tuple(range(9)), + [ + tuple(range(5)), + ] + ] + found = [] + for level, key, val in collect_resolver_dictionary(obj): + found.append(((' ' * level) + key + ': ' + str(val))) + + data_regression.check(found) + + +def _skip_key_in_dict(key): + try: + int(key) + except ValueError: + if 'more' in key or '[' in key: + return False + return True + return False + + +def collect_resolver_dictionary(obj, level=0): + from _pydevd_bundle.pydevd_xml import get_type + resolver = get_type(obj)[-1] + if resolver is None: + return + + dct = resolver.get_dictionary(obj) + for key, val in dct.items(): + if _skip_key_in_dict(key): + continue + yield level, key, val + yield from collect_resolver_dictionary(val, level + 1) diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_dict_multiple_levels_dap_2_.json b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_dict_multiple_levels_dap_2_.json new file mode 100644 index 00000000..4a929818 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_dict_multiple_levels_dap_2_.json @@ -0,0 +1,10 @@ +[ + "1: 1", + "2: {3: 3, 4: 4, 5: 5, 6: 6, 7: {8: 8, 9: 9, 10: 10}}", + " 3: 3", + " 4: 4", + " Unable to handle:: Maximum number of items (2) reached. To show more items customize the value of the PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS environment variable.", + " len(): 5", + "Unable to handle:: Maximum number of items (2) reached. To show more items customize the value of the PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS environment variable.", + "len(): 2" +] \ No newline at end of file diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_dict_multiple_levels_dap_300_.json b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_dict_multiple_levels_dap_300_.json new file mode 100644 index 00000000..70605195 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_dict_multiple_levels_dap_300_.json @@ -0,0 +1,15 @@ +[ + "1: 1", + "2: {3: 3, 4: 4, 5: 5, 6: 6, 7: {8: 8, 9: 9, 10: 10}}", + " 3: 3", + " 4: 4", + " 5: 5", + " 6: 6", + " 7: {8: 8, 9: 9, 10: 10}", + " 8: 8", + " 9: 9", + " 10: 10", + " len(): 3", + " len(): 5", + "len(): 2" +] \ No newline at end of file diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_dap_2_.json b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_dap_2_.json new file mode 100644 index 00000000..3ad1df49 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_dap_2_.json @@ -0,0 +1,28 @@ +[ + "0: (0, 1, 2, 3, 4, 5, 6, 7, 8)", + " 0: 0", + " 1: 1", + " 2: 2", + " 3: 3", + " more: ...", + " [4:6]: [4:6]", + " 4: 4", + " 5: 5", + " [6:8]: [6:8]", + " 6: 6", + " 7: 7", + " [8:9]: [8:9]", + " 8: 8", + " len(): 9", + "1: [(0, 1, 2, 3, 4)]", + " 0: (0, 1, 2, 3, 4)", + " 0: 0", + " 1: 1", + " 2: 2", + " 3: 3", + " more: [4:5]", + " 4: 4", + " len(): 5", + " len(): 1", + "len(): 2" +] \ No newline at end of file diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_dap_300_.json b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_dap_300_.json new file mode 100644 index 00000000..1ac0f2f1 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_dap_300_.json @@ -0,0 +1,23 @@ +[ + "0: (0, 1, 2, 3, 4, 5, 6, 7, 8)", + " 0: 0", + " 1: 1", + " 2: 2", + " 3: 3", + " 4: 4", + " 5: 5", + " 6: 6", + " 7: 7", + " 8: 8", + " len(): 9", + "1: [(0, 1, 2, 3, 4)]", + " 0: (0, 1, 2, 3, 4)", + " 0: 0", + " 1: 1", + " 2: 2", + " 3: 3", + " 4: 4", + " len(): 5", + " len(): 1", + "len(): 2" +] \ No newline at end of file diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_resolver_2_.json b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_resolver_2_.json new file mode 100644 index 00000000..cc1a3714 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_resolver_2_.json @@ -0,0 +1,25 @@ +[ + "0: (0, 1, 2, 3, 4, 5, 6, 7, 8)", + " 0: 0", + " 1: 1", + " 2: 2", + " 3: 3", + " more: ...", + " [4:6]: [4:6]", + " 4: 4", + " 5: 5", + " [6:8]: [6:8]", + " 6: 6", + " 7: 7", + " [8:9]: [8:9]", + " 8: 8", + "1: [(0, 1, 2, 3, 4)]", + " 0: (0, 1, 2, 3, 4)", + " 0: 0", + " 1: 1", + " 2: 2", + " 3: 3", + " more: ...", + " [4:5]: [4:5]", + " 4: 4" +] \ No newline at end of file diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_resolver_300_.json b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_resolver_300_.json new file mode 100644 index 00000000..9469a987 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/tests_python/test_resolvers/test_get_child_variables_multiple_levels_resolver_300_.json @@ -0,0 +1,19 @@ +[ + "0: (0, 1, 2, 3, 4, 5, 6, 7, 8)", + " 0: 0", + " 1: 1", + " 2: 2", + " 3: 3", + " 4: 4", + " 5: 5", + " 6: 6", + " 7: 7", + " 8: 8", + "1: [(0, 1, 2, 3, 4)]", + " 0: (0, 1, 2, 3, 4)", + " 0: 0", + " 1: 1", + " 2: 2", + " 3: 3", + " 4: 4" +] \ No newline at end of file diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_suspended_frames_manager.py b/src/debugpy/_vendored/pydevd/tests_python/test_suspended_frames_manager.py index 6c89b17b..5b63c603 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/test_suspended_frames_manager.py +++ b/src/debugpy/_vendored/pydevd/tests_python/test_suspended_frames_manager.py @@ -1,7 +1,9 @@ import sys from _pydevd_bundle.pydevd_constants import int_types, GENERATED_LEN_ATTR_NAME -from _pydevd_bundle.pydevd_resolver import MAX_ITEMS_TO_HANDLE, TOO_LARGE_ATTR +from _pydevd_bundle.pydevd_resolver import TOO_LARGE_ATTR +from _pydevd_bundle import pydevd_resolver, pydevd_constants from _pydevd_bundle import pydevd_frame_utils +import pytest def get_frame(): @@ -82,28 +84,20 @@ def test_suspended_frames_manager(): }) -_NUMBER_OF_ITEMS_TO_CREATE = MAX_ITEMS_TO_HANDLE + 300 - - def get_dict_large_frame(): obj = {} - for idx in range(_NUMBER_OF_ITEMS_TO_CREATE): + for idx in range(pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS + +300): obj[idx] = (1) return sys._getframe() def get_set_large_frame(): obj = set() - for idx in range(_NUMBER_OF_ITEMS_TO_CREATE): + for idx in range(pydevd_constants.PYDEVD_CONTAINER_RANDOM_ACCESS_MAX_ITEMS + +300): obj.add(idx) return sys._getframe() -def get_tuple_large_frame(): - obj = tuple(range(_NUMBER_OF_ITEMS_TO_CREATE)) - return sys._getframe() - - def test_get_child_variables(): from _pydevd_bundle.pydevd_suspended_frames import SuspendedFramesManager suspended_frames_manager = SuspendedFramesManager() @@ -111,7 +105,6 @@ def test_get_child_variables(): for frame in ( get_dict_large_frame(), get_set_large_frame(), - get_tuple_large_frame(), ): with suspended_frames_manager.track_frames(py_db) as tracker: # : :type tracker: _FramesTracker @@ -123,7 +116,6 @@ def test_get_child_variables(): variable = suspended_frames_manager.get_variable(id(frame)) children_variables = variable.get_child_variable_named('obj').get_children_variables() - assert len(children_variables) < _NUMBER_OF_ITEMS_TO_CREATE found_too_large = False found_len = False