From 10c16fd505be37f4bbfb65eb8ad1c69e6a368ad5 Mon Sep 17 00:00:00 2001 From: Pavel Minaev Date: Wed, 21 Mar 2018 03:20:43 -0700 Subject: [PATCH] Fix #17: Raw value retrieval Add support for rawString formatting. Avoid race conditions over shared state when changing value format. --- ptvsd/futures.py | 13 ++- ptvsd/safe_repr.py | 59 +++++++------ ptvsd/wrapper.py | 138 +++++++++++++++++++++---------- tests/highlevel/test_messages.py | 3 + tests/ptvsd/test_safe_repr.py | 19 +++++ 5 files changed, 156 insertions(+), 76 deletions(-) diff --git a/ptvsd/futures.py b/ptvsd/futures.py index 29b272c4..146580a0 100644 --- a/ptvsd/futures.py +++ b/ptvsd/futures.py @@ -26,10 +26,15 @@ class Future(object): self._exc_info = None def __del__(self): - with self._lock: - if self._done and self._exc_info and not self._observed: - print('Unobserved exception in a Future:', file=sys.__stderr__) - traceback.print_exception(*self._exc_info, file=sys.__stderr__) + if self._lock: + with self._lock: + if self._done and self._exc_info and not self._observed: + print( + 'Unobserved exception in a Future:', + file=sys.__stderr__) + traceback.print_exception( + *self._exc_info, + file=sys.__stderr__) def result(self): # TODO: docstring diff --git a/ptvsd/safe_repr.py b/ptvsd/safe_repr.py index 4b90a94b..42eda021 100644 --- a/ptvsd/safe_repr.py +++ b/ptvsd/safe_repr.py @@ -69,16 +69,19 @@ class SafeRepr(object): maxother_outer = 2 ** 16 maxother_inner = 30 - def __call__(self, obj, convert_to_hex=False): + convert_to_hex = False + raw_value = False + + def __call__(self, obj): try: - return ''.join(self._repr(obj, 0, convert_to_hex=convert_to_hex)) + return ''.join(self._repr(obj, 0)) except Exception: try: return 'An exception was raised: %r' % sys.exc_info()[1] except Exception: return 'An exception was raised' - def _repr(self, obj, level, convert_to_hex=False): + def _repr(self, obj, level): '''Returns an iterable of the parts in the final repr string.''' try: @@ -95,24 +98,21 @@ class SafeRepr(object): for t, prefix, suffix, comma in self.collection_types: if isinstance(obj, t) and has_obj_repr(t): - return self._repr_iter(obj, level, prefix, suffix, comma, - convert_to_hex=convert_to_hex) + return self._repr_iter(obj, level, prefix, suffix, comma) for t, prefix, suffix, item_prefix, item_sep, item_suffix in self.dict_types: # noqa if isinstance(obj, t) and has_obj_repr(t): return self._repr_dict(obj, level, prefix, suffix, - item_prefix, item_sep, item_suffix, - convert_to_hex=convert_to_hex) + item_prefix, item_sep, item_suffix) for t in self.string_types: if isinstance(obj, t) and has_obj_repr(t): - return self._repr_str(obj, level, - convert_to_hex=convert_to_hex) + return self._repr_str(obj, level) if self._is_long_iter(obj): - return self._repr_long_iter(obj, convert_to_hex=convert_to_hex) + return self._repr_long_iter(obj) - return self._repr_other(obj, level, convert_to_hex=convert_to_hex) + return self._repr_other(obj, level) # Determines whether an iterable exceeds the limits set in # maxlimits, and is therefore unsafe to repr(). @@ -169,7 +169,7 @@ class SafeRepr(object): return True def _repr_iter(self, obj, level, prefix, suffix, - comma_after_single_element=False, convert_to_hex=False): + comma_after_single_element=False): yield prefix if level >= len(self.maxcollection): @@ -187,7 +187,7 @@ class SafeRepr(object): yield '...' break - for p in self._repr(item, 100 if item is obj else level + 1, convert_to_hex=convert_to_hex): # noqa + for p in self._repr(item, 100 if item is obj else level + 1): yield p else: if comma_after_single_element: @@ -195,9 +195,9 @@ class SafeRepr(object): yield ',' yield suffix - def _repr_long_iter(self, obj, convert_to_hex=False): + def _repr_long_iter(self, obj): try: - length = hex(len(obj)) if convert_to_hex else len(obj) + length = hex(len(obj)) if self.convert_to_hex else len(obj) obj_repr = '<%s, len() = %s>' % (type(obj).__name__, length) except Exception: try: @@ -207,7 +207,7 @@ class SafeRepr(object): yield obj_repr def _repr_dict(self, obj, level, prefix, suffix, - item_prefix, item_sep, item_suffix, convert_to_hex=False): + item_prefix, item_sep, item_suffix): if not obj: yield prefix + suffix return @@ -236,7 +236,7 @@ class SafeRepr(object): break yield item_prefix - for p in self._repr(key, level + 1, convert_to_hex=convert_to_hex): + for p in self._repr(key, level + 1): yield p yield item_sep @@ -246,26 +246,31 @@ class SafeRepr(object): except Exception: yield '' else: - for p in self._repr(item, 100 if item is obj else level + 1, convert_to_hex=convert_to_hex): # noqa + for p in self._repr(item, 100 if item is obj else level + 1): yield p yield item_suffix yield suffix - def _repr_str(self, obj, level, convert_to_hex=False): + def _repr_str(self, obj, level): return self._repr_obj(obj, level, - self.maxstring_inner, self.maxstring_outer, - convert_to_hex=convert_to_hex) + self.maxstring_inner, self.maxstring_outer) - def _repr_other(self, obj, level, convert_to_hex=False): + def _repr_other(self, obj, level): return self._repr_obj(obj, level, - self.maxother_inner, self.maxother_outer, - convert_to_hex=convert_to_hex) + self.maxother_inner, self.maxother_outer) - def _repr_obj(self, obj, level, limit_inner, limit_outer, - convert_to_hex=False): + def _repr_obj(self, obj, level, limit_inner, limit_outer): try: - if isinstance(obj, self.int_types) and convert_to_hex: + if self.raw_value: + try: + mv = memoryview(obj) + except Exception: + obj_repr = str(obj) + else: + # Map bytes to Unicode codepoints with same values. + obj_repr = mv.tobytes().decode('latin-1') + elif self.convert_to_hex and isinstance(obj, self.int_types): obj_repr = hex(obj) else: obj_repr = repr(obj) diff --git a/ptvsd/wrapper.py b/ptvsd/wrapper.py index e6a68c99..c162159a 100644 --- a/ptvsd/wrapper.py +++ b/ptvsd/wrapper.py @@ -5,6 +5,7 @@ from __future__ import print_function, absolute_import import atexit +import contextlib import errno import os import platform @@ -33,6 +34,7 @@ import ptvsd.ipcjson as ipcjson # noqa import ptvsd.futures as futures # noqa import ptvsd.untangle as untangle # noqa from ptvsd.pathutils import PathUnNormcase # noqa +from ptvsd.safe_repr import SafeRepr # noqa __author__ = "Microsoft Corporation " @@ -54,26 +56,47 @@ class SafeReprPresentationProvider(pydevd_extapi.StrPresentationProvider): to SafeRepr. """ + _lock = threading.Lock() + def __init__(self): - from ptvsd.safe_repr import SafeRepr - self.safe_repr = SafeRepr() - self.convert_to_hex = False + self.set_format({}) def can_provide(self, type_object, type_name): + """Implements StrPresentationProvider.""" return True def get_str(self, val): - return self.safe_repr(val, self.convert_to_hex) + """Implements StrPresentationProvider.""" + return self._repr(val) def set_format(self, fmt): - self.convert_to_hex = fmt['hex'] + """ + Use fmt for all future formatting operations done by this provider. + """ + safe_repr = SafeRepr() + safe_repr.convert_to_hex = fmt.get('hex', False) + safe_repr.raw_value = fmt.get('rawString', False) + self._repr = safe_repr + @contextlib.contextmanager + def using_format(self, fmt): + """ + Returns a context manager that invokes set_format(fmt) on enter, + and restores the old format on exit. + """ + old_repr = self._repr + self.set_format(fmt) + yield + self._repr = old_repr + + +# Do not access directly - use safe_repr_provider() instead! +SafeReprPresentationProvider._instance = SafeReprPresentationProvider() # Register our presentation provider as the first item on the list, # so that we're in full control of presentation. str_handlers = pydevd_extutil.EXTENSION_MANAGER_INSTANCE.type_to_instance.setdefault(pydevd_extapi.StrPresentationProvider, []) # noqa -safe_repr_provider = SafeReprPresentationProvider() -str_handlers.insert(0, safe_repr_provider) +str_handlers.insert(0, SafeReprPresentationProvider._instance) class UnsupportedPyDevdCommandError(Exception): @@ -594,8 +617,17 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): raise UnsupportedPyDevdCommandError(cmd_id) return f(self, seq, args) + def async_method(m): + """Converts a generator method into an async one.""" + m = futures.wrap_async(m) + + def f(self, *args, **kwargs): + return m(self, self.loop, *args, **kwargs) + + return f + def async_handler(m): - # TODO: docstring + """Converts a generator method into a fire-and-forget async one.""" m = futures.wrap_async(m) def f(self, *args, **kwargs): @@ -611,6 +643,24 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): return f + def sleep(self): + fut = futures.Future(self.loop) + self.loop.call_soon(lambda: fut.set_result(None)) + return fut + + @async_method + def using_format(self, fmt): + while not SafeReprPresentationProvider._lock.acquire(False): + yield self.sleep() + provider = SafeReprPresentationProvider._instance + + @contextlib.contextmanager + def context(): + with provider.using_format(fmt): + yield + provider._lock.release() + yield futures.Result(context()) + @async_handler def on_initialize(self, request, args): # TODO: docstring @@ -807,19 +857,14 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): } self.send_response(request, scopes=[scope]) - def _extract_format(self, args): - fmt = args.get('format', {}) - fmt.setdefault('hex', False) - return fmt - @async_handler def on_variables(self, request, args): - # TODO: docstring - vsc_var = int(args['variablesReference']) - pyd_var = self.var_map.to_pydevd(vsc_var) + """Handles DAP VariablesRequest.""" - safe_repr_provider.set_format( - self._extract_format(args)) + vsc_var = int(args['variablesReference']) + fmt = args.get('format', {}) + + pyd_var = self.var_map.to_pydevd(vsc_var) if len(pyd_var) == 3: cmd = pydevd_comm.CMD_GET_FRAME @@ -827,7 +872,9 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): cmd = pydevd_comm.CMD_GET_VARIABLE cmdargs = (str(s) for s in pyd_var) msg = '\t'.join(cmdargs) - _, _, resp_args = yield self.pydevd_request(cmd, msg) + with (yield self.using_format(fmt)): + _, _, resp_args = yield self.pydevd_request(cmd, msg) + xml = untangle.parse(resp_args).xml try: xvars = xml.var @@ -845,6 +892,9 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): 'value': var_value, } + if var_type in ('str', 'unicode', 'bytes', 'bytearray'): + var['presentationHint'] = {'attributes': ['rawString']} + if bool(xvar['isContainer']): pyd_child = pyd_var + (var_name,) var['variablesReference'] = self.var_map.to_vscode( @@ -856,8 +906,6 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): variables.append(var) - # Reset hex format since this is per request. - safe_repr_provider.convert_to_hex = False self.send_response(request, variables=variables.get_sorted_variables()) def __get_variable_evaluate_name(self, pyd_var_parent, var_name): @@ -899,11 +947,14 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): @async_handler def on_setVariable(self, request, args): + """Handles DAP SetVariableRequest.""" + vsc_var = int(args['variablesReference']) pyd_var = self.var_map.to_pydevd(vsc_var) var_name = args['name'] var_value = args['value'] + fmt = args.get('format', {}) lhs_expr = self.__get_variable_evaluate_name(pyd_var, var_name) if not lhs_expr: @@ -915,9 +966,6 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): pyd_tid = str(pyd_var[0]) pyd_fid = str(pyd_var[1]) - safe_repr_provider.set_format( - self._extract_format(args)) - # VSC gives us variablesReference to the parent of the variable # being set, and variable name; but pydevd wants the ID # (or rather path) of the variable itself. @@ -925,16 +973,18 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): vsc_var = self.var_map.to_vscode(pyd_var, autogen=True) cmd_args = [pyd_tid, pyd_fid, 'LOCAL', expr, '1'] - yield self.pydevd_request( - pydevd_comm.CMD_EXEC_EXPRESSION, - '\t'.join(cmd_args), - ) + with (yield self.using_format(fmt)): + yield self.pydevd_request( + pydevd_comm.CMD_EXEC_EXPRESSION, + '\t'.join(cmd_args), + ) cmd_args = [pyd_tid, pyd_fid, 'LOCAL', lhs_expr, '1'] - _, _, resp_args = yield self.pydevd_request( - pydevd_comm.CMD_EVALUATE_EXPRESSION, - '\t'.join(cmd_args), - ) + with (yield self.using_format(fmt)): + _, _, resp_args = yield self.pydevd_request( + pydevd_comm.CMD_EVALUATE_EXPRESSION, + '\t'.join(cmd_args), + ) xml = untangle.parse(resp_args).xml xvar = xml.var @@ -946,26 +996,25 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): if bool(xvar['isContainer']): response['variablesReference'] = vsc_var - # Reset hex format since this is per request. - safe_repr_provider.convert_to_hex = False self.send_response(request, **response) @async_handler def on_evaluate(self, request, args): + """Handles DAP EvaluateRequest.""" + # pydevd message format doesn't permit tabs in expressions expr = args['expression'].replace('\t', ' ') + fmt = args.get('format', {}) vsc_fid = int(args['frameId']) pyd_tid, pyd_fid = self.frame_map.to_pydevd(vsc_fid) - safe_repr_provider.set_format( - self._extract_format(args)) - cmd_args = (pyd_tid, pyd_fid, 'LOCAL', expr, '1') msg = '\t'.join(str(s) for s in cmd_args) - _, _, resp_args = yield self.pydevd_request( - pydevd_comm.CMD_EVALUATE_EXPRESSION, - msg) + with (yield self.using_format(fmt)): + _, _, resp_args = yield self.pydevd_request( + pydevd_comm.CMD_EVALUATE_EXPRESSION, + msg) xml = untangle.parse(resp_args).xml xvar = xml.var @@ -980,9 +1029,10 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): if context == 'repl' and is_eval_error == 'True': # try exec for repl requests - _, _, resp_args = yield self.pydevd_request( - pydevd_comm.CMD_EXEC_EXPRESSION, - msg) + with (yield self.using_format(fmt)): + _, _, resp_args = yield self.pydevd_request( + pydevd_comm.CMD_EXEC_EXPRESSION, + msg) try: xml2 = untangle.parse(resp_args).xml xvar2 = xml2.var @@ -1011,8 +1061,6 @@ class VSCodeMessageProcessor(ipcjson.SocketIO, ipcjson.IpcChannel): if bool(xvar['isContainer']): response['variablesReference'] = vsc_var - # Reset hex format since this is per request. - safe_repr_provider.convert_to_hex = False self.send_response(request, **response) @async_handler diff --git a/tests/highlevel/test_messages.py b/tests/highlevel/test_messages.py index 0a1c860e..a7879035 100644 --- a/tests/highlevel/test_messages.py +++ b/tests/highlevel/test_messages.py @@ -405,6 +405,9 @@ class VariablesTests(NormalRequestTest, unittest.TestCase): 'name': 'spam', 'type': 'str', 'value': "'eggs'", + 'presentationHint': { + 'attributes': ['rawString'], + }, }, { 'name': 'x', diff --git a/tests/ptvsd/test_safe_repr.py b/tests/ptvsd/test_safe_repr.py index afbc9b8e..31fcec52 100644 --- a/tests/ptvsd/test_safe_repr.py +++ b/tests/ptvsd/test_safe_repr.py @@ -184,6 +184,25 @@ class StringTests(TestBase): raise NotImplementedError +class RawValueTests(TestBase): + + def setUp(self): + super(RawValueTests, self).setUp() + self.saferepr.raw_value = True + + def test_unicode_raw(self): + value = u'A' * 5 + self.assert_saferepr(value, 'AAAAA') + + def test_bytes_raw(self): + value = b'A' * 5 + self.assert_saferepr(value, 'AAAAA') + + def test_bytearray_raw(self): + value = bytearray(b'A' * 5) + self.assert_saferepr(value, 'AAAAA') + + class NumberTests(TestBase): @unittest.skip('not written') # TODO: finish!