From 9ab891b3280e511e15ce10ee3a92b02f76b43f21 Mon Sep 17 00:00:00 2001 From: Fabio Zadrozny Date: Thu, 18 Feb 2021 15:38:46 -0300 Subject: [PATCH] Code reloading. Fixes #212 --- cgmanifest.json | 11 + src/debugpy/ThirdPartyNotices.txt | 30 + .../pydevd/_pydev_bundle/fsnotify/__init__.py | 361 +++++++++ .../fsnotify/scandir_vendored.py | 720 ++++++++++++++++++ .../pydevd/_pydev_imps/_pydev_execfile.py | 8 +- .../pydevd/_pydevd_bundle/pydevd_api.py | 12 +- .../pydevd/_pydevd_bundle/pydevd_comm.py | 118 ++- .../_pydevd_bundle/pydevd_dont_trace_files.py | 1 + .../pydevd_net_command_factory_json.py | 4 + .../pydevd_process_net_command.py | 9 +- .../pydevd_process_net_command_json.py | 45 ++ .../pydevd/_pydevd_bundle/pydevd_reload.py | 95 +-- .../pydevd/_pydevd_bundle/pydevd_utils.py | 7 + src/debugpy/_vendored/pydevd/pydevd.py | 80 +- .../pydevd/tests_python/test_debugger.py | 30 +- .../pydevd/tests_python/test_debugger_json.py | 59 +- .../tests_python/test_pydevd_filtering.py | 8 + tests/debug/config.py | 13 + tests/debug/session.py | 6 +- 19 files changed, 1505 insertions(+), 112 deletions(-) create mode 100644 src/debugpy/_vendored/pydevd/_pydev_bundle/fsnotify/__init__.py create mode 100644 src/debugpy/_vendored/pydevd/_pydev_bundle/fsnotify/scandir_vendored.py diff --git a/cgmanifest.json b/cgmanifest.json index 3d0c13a7..0d6c30f7 100644 --- a/cgmanifest.json +++ b/cgmanifest.json @@ -49,6 +49,17 @@ }, "DevelopmentDependency": false } + }, + { + "Component": { + "Type": "git", + "git": { + "Name": "scandir", + "RepositoryUrl": "https://github.com/benhoyt/scandir", + "CommitHash": "6ed381881bc2fb9de05804e892eeeeb3601a3af2" + }, + "DevelopmentDependency": false + } } ] } diff --git a/src/debugpy/ThirdPartyNotices.txt b/src/debugpy/ThirdPartyNotices.txt index 516a8807..2b309900 100644 --- a/src/debugpy/ThirdPartyNotices.txt +++ b/src/debugpy/ThirdPartyNotices.txt @@ -466,4 +466,34 @@ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========================================= +Includes https://github.com/benhoyt/scandir 6ed381881bc2fb9de05804e892eeeeb3601a3af2 - BSD 3-Clause "New" or "Revised" License + +Copyright (c) 2012, Ben Hoyt +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +* Neither the name of Ben Hoyt nor the names of its contributors may be used +to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +========================================= END OF PyDev.Debugger NOTICES, INFORMATION, AND LICENSE diff --git a/src/debugpy/_vendored/pydevd/_pydev_bundle/fsnotify/__init__.py b/src/debugpy/_vendored/pydevd/_pydev_bundle/fsnotify/__init__.py new file mode 100644 index 00000000..b001799e --- /dev/null +++ b/src/debugpy/_vendored/pydevd/_pydev_bundle/fsnotify/__init__.py @@ -0,0 +1,361 @@ +''' +Sample usage to track changes in a thread. + + import threading + import time + watcher = fsnotify.Watcher() + watcher.accepted_file_extensions = {'.py', '.pyw'} + + # Configure target values to compute throttling. + # Note: internal sleep times will be updated based on + # profiling the actual application runtime to match + # those values. + + watcher.target_time_for_single_scan = 2. + watcher.target_time_for_notification = 4. + + watcher.set_tracked_paths([target_dir]) + + def start_watching(): # Called from thread + for change_enum, change_path in watcher.iter_changes(): + if change_enum == fsnotify.Change.added: + print('Added: ', change_path) + elif change_enum == fsnotify.Change.modified: + print('Modified: ', change_path) + elif change_enum == fsnotify.Change.deleted: + print('Deleted: ', change_path) + + t = threading.Thread(target=start_watching) + t.daemon = True + t.start() + + try: + ... + finally: + watcher.dispose() + + +Note: changes are only reported for files (added/modified/deleted), not directories. +''' +import threading +import sys +from os.path import basename +from _pydev_bundle import pydev_log +try: + from os import scandir +except: + try: + # Search an installed version (which may have speedups). + from scandir import scandir + except: + # If all fails, use our vendored version (which won't have speedups). + from .scandir_vendored import scandir + +try: + from enum import IntEnum +except: + + class IntEnum(object): + pass + +import time + +__author__ = 'Fabio Zadrozny' +__email__ = 'fabiofz@gmail.com' +__version__ = '0.1.5' # Version here and in setup.py + + +class Change(IntEnum): + added = 1 + modified = 2 + deleted = 3 + + +class _SingleVisitInfo(object): + + def __init__(self): + self.count = 0 + self.visited_dirs = set() + self.file_to_mtime = {} + self.last_sleep_time = time.time() + + +class _PathWatcher(object): + ''' + Helper to watch a single path. + ''' + + def __init__(self, root_path, accept_directory, accept_file, single_visit_info, max_recursion_level, sleep_time=.0): + ''' + :type root_path: str + :type accept_directory: Callback[str, bool] + :type accept_file: Callback[str, bool] + :type max_recursion_level: int + :type sleep_time: float + ''' + self.accept_directory = accept_directory + self.accept_file = accept_file + self._max_recursion_level = max_recursion_level + + self._root_path = root_path + + # Initial sleep value for throttling, it'll be auto-updated based on the + # Watcher.target_time_for_single_scan. + self.sleep_time = sleep_time + + self.sleep_at_elapsed = 1. / 30. + + # When created, do the initial snapshot right away! + old_file_to_mtime = {} + self._check(single_visit_info, lambda _change: None, old_file_to_mtime) + + def __eq__(self, o): + if isinstance(o, _PathWatcher): + return self._root_path == o._root_path + + return False + + def __ne__(self, o): + return not self == o + + def __hash__(self): + return hash(self._root_path) + + def _check_dir(self, dir_path, single_visit_info, append_change, old_file_to_mtime, level): + # This is the actual poll loop + if dir_path in single_visit_info.visited_dirs or level > self._max_recursion_level: + return + single_visit_info.visited_dirs.add(dir_path) + try: + if isinstance(dir_path, bytes): + try: + dir_path = dir_path.decode(sys.getfilesystemencoding()) + except UnicodeDecodeError: + try: + dir_path = dir_path.decode('utf-8') + except UnicodeDecodeError: + return # Ignore if we can't deal with the path. + + new_files = single_visit_info.file_to_mtime + + for entry in scandir(dir_path): + single_visit_info.count += 1 + + # Throttle if needed inside the loop + # to avoid consuming too much CPU. + if single_visit_info.count % 300 == 0: + if self.sleep_time > 0: + t = time.time() + diff = t - single_visit_info.last_sleep_time + if diff > self.sleep_at_elapsed: + time.sleep(self.sleep_time) + single_visit_info.last_sleep_time = time.time() + + if entry.is_dir(): + if self.accept_directory(entry.path): + self._check_dir(entry.path, single_visit_info, append_change, old_file_to_mtime, level + 1) + + elif self.accept_file(entry.path): + stat = entry.stat() + mtime = (stat.st_mtime_ns, stat.st_size) + path = entry.path + new_files[path] = mtime + + old_mtime = old_file_to_mtime.pop(path, None) + if not old_mtime: + append_change((Change.added, path)) + elif old_mtime != mtime: + append_change((Change.modified, path)) + + except OSError: + pass # Directory was removed in the meanwhile. + + def _check(self, single_visit_info, append_change, old_file_to_mtime): + self._check_dir(self._root_path, single_visit_info, append_change, old_file_to_mtime, 0) + + +class Watcher(object): + + # By default (if accept_directory is not specified), these will be the + # ignored directories. + ignored_dirs = {u'.git', u'__pycache__', u'.idea', u'node_modules', u'.metadata'} + + # By default (if accept_file is not specified), these will be the + # accepted files. + accepted_file_extensions = () + + # Set to the target value for doing full scan of all files (adds a sleep inside the poll loop + # which processes files to reach the target time). + # Lower values will consume more CPU + # Set to 0.0 to have no sleeps (which will result in a higher cpu load). + target_time_for_single_scan = 2.0 + + # Set the target value from the start of one scan to the start of another scan (adds a + # sleep after a full poll is done to reach the target time). + # Lower values will consume more CPU. + # Set to 0.0 to have a new scan start right away without any sleeps. + target_time_for_notification = 4.0 + + # Set to True to print the time for a single poll through all the paths. + print_poll_time = False + + # This is the maximum recursion level. + max_recursion_level = 10 + + def __init__(self, accept_directory=None, accept_file=None): + ''' + :param Callable[str, bool] accept_directory: + Callable that returns whether a directory should be watched. + Note: if passed it'll override the `ignored_dirs` + + :param Callable[str, bool] accept_file: + Callable that returns whether a file should be watched. + Note: if passed it'll override the `accepted_file_extensions`. + ''' + self._path_watchers = set() + self._disposed = threading.Event() + + if accept_directory is None: + accept_directory = lambda dir_path: basename(dir_path) not in self.ignored_dirs + if accept_file is None: + accept_file = lambda path_name: \ + not self.accepted_file_extensions or path_name.endswith(self.accepted_file_extensions) + self.accept_file = accept_file + self.accept_directory = accept_directory + self._single_visit_info = _SingleVisitInfo() + + @property + def accept_directory(self): + return self._accept_directory + + @accept_directory.setter + def accept_directory(self, accept_directory): + self._accept_directory = accept_directory + for path_watcher in self._path_watchers: + path_watcher.accept_directory = accept_directory + + @property + def accept_file(self): + return self._accept_file + + @accept_file.setter + def accept_file(self, accept_file): + self._accept_file = accept_file + for path_watcher in self._path_watchers: + path_watcher.accept_file = accept_file + + def dispose(self): + self._disposed.set() + + @property + def path_watchers(self): + return tuple(self._path_watchers) + + def set_tracked_paths(self, paths): + """ + Note: always resets all path trackers to track the passed paths. + """ + if not isinstance(paths, (list, tuple, set)): + paths = (paths,) + + # Sort by the path len so that the bigger paths come first (so, + # if there's any nesting we want the nested paths to be visited + # before the parent paths so that the max_recursion_level is correct). + paths = sorted(set(paths), key=lambda path: -len(path)) + path_watchers = set() + + self._single_visit_info = _SingleVisitInfo() + + initial_time = time.time() + for path in paths: + sleep_time = 0. # When collecting the first time, sleep_time should be 0! + path_watcher = _PathWatcher( + path, + self.accept_directory, + self.accept_file, + self._single_visit_info, + max_recursion_level=self.max_recursion_level, + sleep_time=sleep_time, + ) + + path_watchers.add(path_watcher) + + actual_time = (time.time() - initial_time) + + pydev_log.debug('Tracking the following paths for changes: %s', paths) + pydev_log.debug('Time to track: %.2fs', actual_time) + pydev_log.debug('Folders found: %s', len(self._single_visit_info.visited_dirs)) + pydev_log.debug('Files found: %s', len(self._single_visit_info.file_to_mtime)) + self._path_watchers = path_watchers + + def iter_changes(self): + ''' + Continuously provides changes (until dispose() is called). + + Changes provided are tuples with the Change enum and filesystem path. + + :rtype: Iterable[Tuple[Change, str]] + ''' + while not self._disposed.is_set(): + initial_time = time.time() + + old_visit_info = self._single_visit_info + old_file_to_mtime = old_visit_info.file_to_mtime + changes = [] + append_change = changes.append + + self._single_visit_info = single_visit_info = _SingleVisitInfo() + for path_watcher in self._path_watchers: + path_watcher._check(single_visit_info, append_change, old_file_to_mtime) + + # Note that we pop entries while visiting, so, what remained is what's deleted. + for entry in old_file_to_mtime: + append_change((Change.deleted, entry)) + + for change in changes: + yield change + + actual_time = (time.time() - initial_time) + if self.print_poll_time: + print('--- Total poll time: %.3fs' % actual_time) + + if actual_time > 0: + if self.target_time_for_single_scan <= 0.0: + for path_watcher in self._path_watchers: + path_watcher.sleep_time = 0.0 + else: + perc = self.target_time_for_single_scan / actual_time + + # Prevent from changing the values too much (go slowly into the right + # direction). + # (to prevent from cases where the user puts the machine on sleep and + # values become too skewed). + if perc > 2.: + perc = 2. + elif perc < 0.5: + perc = 0.5 + + for path_watcher in self._path_watchers: + if path_watcher.sleep_time <= 0.0: + path_watcher.sleep_time = 0.001 + new_sleep_time = path_watcher.sleep_time * perc + + # Prevent from changing the values too much (go slowly into the right + # direction). + # (to prevent from cases where the user puts the machine on sleep and + # values become too skewed). + diff_sleep_time = new_sleep_time - path_watcher.sleep_time + path_watcher.sleep_time += (diff_sleep_time / (3.0 * len(self._path_watchers))) + + if actual_time > 0: + self._disposed.wait(actual_time) + + if path_watcher.sleep_time < 0.001: + path_watcher.sleep_time = 0.001 + + # print('new sleep time: %s' % path_watcher.sleep_time) + + diff = self.target_time_for_notification - actual_time + if diff > 0.: + self._disposed.wait(diff) + diff --git a/src/debugpy/_vendored/pydevd/_pydev_bundle/fsnotify/scandir_vendored.py b/src/debugpy/_vendored/pydevd/_pydev_bundle/fsnotify/scandir_vendored.py new file mode 100644 index 00000000..e38df9c7 --- /dev/null +++ b/src/debugpy/_vendored/pydevd/_pydev_bundle/fsnotify/scandir_vendored.py @@ -0,0 +1,720 @@ +"""scandir, a better directory iterator and faster os.walk(), now in the Python 3.5 stdlib + +scandir() is a generator version of os.listdir() that returns an +iterator over files in a directory, and also exposes the extra +information most OSes provide while iterating files in a directory +(such as type and stat information). + +This module also includes a version of os.walk() that uses scandir() +to speed it up significantly. + +See README.md or https://github.com/benhoyt/scandir for rationale and +docs, or read PEP 471 (https://www.python.org/dev/peps/pep-0471/) for +more details on its inclusion into Python 3.5 + +scandir is released under the new BSD 3-clause license. + +Copyright (c) 2012, Ben Hoyt +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +* Neither the name of Ben Hoyt nor the names of its contributors may be used +to endorse or promote products derived from this software without specific +prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" + +from __future__ import division + +from errno import ENOENT +from os import listdir, lstat, stat, strerror +from os.path import join, islink +from stat import S_IFDIR, S_IFLNK, S_IFREG +import collections +import sys + +try: + import _scandir +except ImportError: + _scandir = None + +try: + import ctypes +except ImportError: + ctypes = None + +if _scandir is None and ctypes is None: + import warnings + warnings.warn("scandir can't find the compiled _scandir C module " + "or ctypes, using slow generic fallback") + +__version__ = '1.10.0' +__all__ = ['scandir', 'walk'] + +# Windows FILE_ATTRIBUTE constants for interpreting the +# FIND_DATA.dwFileAttributes member +FILE_ATTRIBUTE_ARCHIVE = 32 +FILE_ATTRIBUTE_COMPRESSED = 2048 +FILE_ATTRIBUTE_DEVICE = 64 +FILE_ATTRIBUTE_DIRECTORY = 16 +FILE_ATTRIBUTE_ENCRYPTED = 16384 +FILE_ATTRIBUTE_HIDDEN = 2 +FILE_ATTRIBUTE_INTEGRITY_STREAM = 32768 +FILE_ATTRIBUTE_NORMAL = 128 +FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 8192 +FILE_ATTRIBUTE_NO_SCRUB_DATA = 131072 +FILE_ATTRIBUTE_OFFLINE = 4096 +FILE_ATTRIBUTE_READONLY = 1 +FILE_ATTRIBUTE_REPARSE_POINT = 1024 +FILE_ATTRIBUTE_SPARSE_FILE = 512 +FILE_ATTRIBUTE_SYSTEM = 4 +FILE_ATTRIBUTE_TEMPORARY = 256 +FILE_ATTRIBUTE_VIRTUAL = 65536 + +IS_PY3 = sys.version_info >= (3, 0) + +if IS_PY3: + unicode = str # Because Python <= 3.2 doesn't have u'unicode' syntax + + +class GenericDirEntry(object): + __slots__ = ('name', '_stat', '_lstat', '_scandir_path', '_path') + + def __init__(self, scandir_path, name): + self._scandir_path = scandir_path + self.name = name + self._stat = None + self._lstat = None + self._path = None + + @property + def path(self): + if self._path is None: + self._path = join(self._scandir_path, self.name) + return self._path + + def stat(self, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + self._stat = stat(self.path) + return self._stat + else: + if self._lstat is None: + self._lstat = lstat(self.path) + return self._lstat + + # The code duplication below is intentional: this is for slightly + # better performance on systems that fall back to GenericDirEntry. + # It avoids an additional attribute lookup and method call, which + # are relatively slow on CPython. + def is_dir(self, follow_symlinks=True): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False # Path doesn't exist or is a broken symlink + return st.st_mode & 0o170000 == S_IFDIR + + def is_file(self, follow_symlinks=True): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False # Path doesn't exist or is a broken symlink + return st.st_mode & 0o170000 == S_IFREG + + def is_symlink(self): + try: + st = self.stat(follow_symlinks=False) + except OSError as e: + if e.errno != ENOENT: + raise + return False # Path doesn't exist or is a broken symlink + return st.st_mode & 0o170000 == S_IFLNK + + def inode(self): + st = self.stat(follow_symlinks=False) + return st.st_ino + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + +def _scandir_generic(path=unicode('.')): + """Like os.listdir(), but yield DirEntry objects instead of returning + a list of names. + """ + for name in listdir(path): + yield GenericDirEntry(path, name) + + +if IS_PY3 and sys.platform == 'win32': + def scandir_generic(path=unicode('.')): + if isinstance(path, bytes): + raise TypeError("os.scandir() doesn't support bytes path on Windows, use Unicode instead") + return _scandir_generic(path) + scandir_generic.__doc__ = _scandir_generic.__doc__ +else: + scandir_generic = _scandir_generic + + +scandir_c = None +scandir_python = None + + +if sys.platform == 'win32': + if ctypes is not None: + from ctypes import wintypes + + # Various constants from windows.h + INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value + ERROR_FILE_NOT_FOUND = 2 + ERROR_NO_MORE_FILES = 18 + IO_REPARSE_TAG_SYMLINK = 0xA000000C + + # Numer of seconds between 1601-01-01 and 1970-01-01 + SECONDS_BETWEEN_EPOCHS = 11644473600 + + kernel32 = ctypes.windll.kernel32 + + # ctypes wrappers for (wide string versions of) FindFirstFile, + # FindNextFile, and FindClose + FindFirstFile = kernel32.FindFirstFileW + FindFirstFile.argtypes = [ + wintypes.LPCWSTR, + ctypes.POINTER(wintypes.WIN32_FIND_DATAW), + ] + FindFirstFile.restype = wintypes.HANDLE + + FindNextFile = kernel32.FindNextFileW + FindNextFile.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(wintypes.WIN32_FIND_DATAW), + ] + FindNextFile.restype = wintypes.BOOL + + FindClose = kernel32.FindClose + FindClose.argtypes = [wintypes.HANDLE] + FindClose.restype = wintypes.BOOL + + Win32StatResult = collections.namedtuple('Win32StatResult', [ + 'st_mode', + 'st_ino', + 'st_dev', + 'st_nlink', + 'st_uid', + 'st_gid', + 'st_size', + 'st_atime', + 'st_mtime', + 'st_ctime', + 'st_atime_ns', + 'st_mtime_ns', + 'st_ctime_ns', + 'st_file_attributes', + ]) + + def filetime_to_time(filetime): + """Convert Win32 FILETIME to time since Unix epoch in seconds.""" + total = filetime.dwHighDateTime << 32 | filetime.dwLowDateTime + return total / 10000000 - SECONDS_BETWEEN_EPOCHS + + def find_data_to_stat(data): + """Convert Win32 FIND_DATA struct to stat_result.""" + # First convert Win32 dwFileAttributes to st_mode + attributes = data.dwFileAttributes + st_mode = 0 + if attributes & FILE_ATTRIBUTE_DIRECTORY: + st_mode |= S_IFDIR | 0o111 + else: + st_mode |= S_IFREG + if attributes & FILE_ATTRIBUTE_READONLY: + st_mode |= 0o444 + else: + st_mode |= 0o666 + if (attributes & FILE_ATTRIBUTE_REPARSE_POINT and + data.dwReserved0 == IO_REPARSE_TAG_SYMLINK): + st_mode ^= st_mode & 0o170000 + st_mode |= S_IFLNK + + st_size = data.nFileSizeHigh << 32 | data.nFileSizeLow + st_atime = filetime_to_time(data.ftLastAccessTime) + st_mtime = filetime_to_time(data.ftLastWriteTime) + st_ctime = filetime_to_time(data.ftCreationTime) + + # Some fields set to zero per CPython's posixmodule.c: st_ino, st_dev, + # st_nlink, st_uid, st_gid + return Win32StatResult(st_mode, 0, 0, 0, 0, 0, st_size, + st_atime, st_mtime, st_ctime, + int(st_atime * 1000000000), + int(st_mtime * 1000000000), + int(st_ctime * 1000000000), + attributes) + + class Win32DirEntryPython(object): + __slots__ = ('name', '_stat', '_lstat', '_find_data', '_scandir_path', '_path', '_inode') + + def __init__(self, scandir_path, name, find_data): + self._scandir_path = scandir_path + self.name = name + self._stat = None + self._lstat = None + self._find_data = find_data + self._path = None + self._inode = None + + @property + def path(self): + if self._path is None: + self._path = join(self._scandir_path, self.name) + return self._path + + def stat(self, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + if self.is_symlink(): + # It's a symlink, call link-following stat() + self._stat = stat(self.path) + else: + # Not a symlink, stat is same as lstat value + if self._lstat is None: + self._lstat = find_data_to_stat(self._find_data) + self._stat = self._lstat + return self._stat + else: + if self._lstat is None: + # Lazily convert to stat object, because it's slow + # in Python, and often we only need is_dir() etc + self._lstat = find_data_to_stat(self._find_data) + return self._lstat + + def is_dir(self, follow_symlinks=True): + is_symlink = self.is_symlink() + if follow_symlinks and is_symlink: + try: + return self.stat().st_mode & 0o170000 == S_IFDIR + except OSError as e: + if e.errno != ENOENT: + raise + return False + elif is_symlink: + return False + else: + return (self._find_data.dwFileAttributes & + FILE_ATTRIBUTE_DIRECTORY != 0) + + def is_file(self, follow_symlinks=True): + is_symlink = self.is_symlink() + if follow_symlinks and is_symlink: + try: + return self.stat().st_mode & 0o170000 == S_IFREG + except OSError as e: + if e.errno != ENOENT: + raise + return False + elif is_symlink: + return False + else: + return (self._find_data.dwFileAttributes & + FILE_ATTRIBUTE_DIRECTORY == 0) + + def is_symlink(self): + return (self._find_data.dwFileAttributes & + FILE_ATTRIBUTE_REPARSE_POINT != 0 and + self._find_data.dwReserved0 == IO_REPARSE_TAG_SYMLINK) + + def inode(self): + if self._inode is None: + self._inode = lstat(self.path).st_ino + return self._inode + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + def win_error(error, filename): + exc = WindowsError(error, ctypes.FormatError(error)) + exc.filename = filename + return exc + + def _scandir_python(path=unicode('.')): + """Like os.listdir(), but yield DirEntry objects instead of returning + a list of names. + """ + # Call FindFirstFile and handle errors + if isinstance(path, bytes): + is_bytes = True + filename = join(path.decode('mbcs', 'strict'), '*.*') + else: + is_bytes = False + filename = join(path, '*.*') + data = wintypes.WIN32_FIND_DATAW() + data_p = ctypes.byref(data) + handle = FindFirstFile(filename, data_p) + if handle == INVALID_HANDLE_VALUE: + error = ctypes.GetLastError() + if error == ERROR_FILE_NOT_FOUND: + # No files, don't yield anything + return + raise win_error(error, path) + + # Call FindNextFile in a loop, stopping when no more files + try: + while True: + # Skip '.' and '..' (current and parent directory), but + # otherwise yield (filename, stat_result) tuple + name = data.cFileName + if name not in ('.', '..'): + if is_bytes: + name = name.encode('mbcs', 'replace') + yield Win32DirEntryPython(path, name, data) + + data = wintypes.WIN32_FIND_DATAW() + data_p = ctypes.byref(data) + success = FindNextFile(handle, data_p) + if not success: + error = ctypes.GetLastError() + if error == ERROR_NO_MORE_FILES: + break + raise win_error(error, path) + finally: + if not FindClose(handle): + raise win_error(ctypes.GetLastError(), path) + + if IS_PY3: + def scandir_python(path=unicode('.')): + if isinstance(path, bytes): + raise TypeError("os.scandir() doesn't support bytes path on Windows, use Unicode instead") + return _scandir_python(path) + scandir_python.__doc__ = _scandir_python.__doc__ + else: + scandir_python = _scandir_python + + if _scandir is not None: + scandir_c = _scandir.scandir + DirEntry_c = _scandir.DirEntry + + if _scandir is not None: + scandir = scandir_c + DirEntry = DirEntry_c + elif ctypes is not None: + scandir = scandir_python + DirEntry = Win32DirEntryPython + else: + scandir = scandir_generic + DirEntry = GenericDirEntry + + +# Linux, OS X, and BSD implementation +elif sys.platform.startswith(('linux', 'darwin', 'sunos5')) or 'bsd' in sys.platform: + have_dirent_d_type = (sys.platform != 'sunos5') + + if ctypes is not None and have_dirent_d_type: + import ctypes.util + + DIR_p = ctypes.c_void_p + + # Rather annoying how the dirent struct is slightly different on each + # platform. The only fields we care about are d_name and d_type. + class Dirent(ctypes.Structure): + if sys.platform.startswith('linux'): + _fields_ = ( + ('d_ino', ctypes.c_ulong), + ('d_off', ctypes.c_long), + ('d_reclen', ctypes.c_ushort), + ('d_type', ctypes.c_byte), + ('d_name', ctypes.c_char * 256), + ) + elif 'openbsd' in sys.platform: + _fields_ = ( + ('d_ino', ctypes.c_uint64), + ('d_off', ctypes.c_uint64), + ('d_reclen', ctypes.c_uint16), + ('d_type', ctypes.c_uint8), + ('d_namlen', ctypes.c_uint8), + ('__d_padding', ctypes.c_uint8 * 4), + ('d_name', ctypes.c_char * 256), + ) + else: + _fields_ = ( + ('d_ino', ctypes.c_uint32), # must be uint32, not ulong + ('d_reclen', ctypes.c_ushort), + ('d_type', ctypes.c_byte), + ('d_namlen', ctypes.c_byte), + ('d_name', ctypes.c_char * 256), + ) + + DT_UNKNOWN = 0 + DT_DIR = 4 + DT_REG = 8 + DT_LNK = 10 + + Dirent_p = ctypes.POINTER(Dirent) + Dirent_pp = ctypes.POINTER(Dirent_p) + + libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) + opendir = libc.opendir + opendir.argtypes = [ctypes.c_char_p] + opendir.restype = DIR_p + + readdir_r = libc.readdir_r + readdir_r.argtypes = [DIR_p, Dirent_p, Dirent_pp] + readdir_r.restype = ctypes.c_int + + closedir = libc.closedir + closedir.argtypes = [DIR_p] + closedir.restype = ctypes.c_int + + file_system_encoding = sys.getfilesystemencoding() + + class PosixDirEntry(object): + __slots__ = ('name', '_d_type', '_stat', '_lstat', '_scandir_path', '_path', '_inode') + + def __init__(self, scandir_path, name, d_type, inode): + self._scandir_path = scandir_path + self.name = name + self._d_type = d_type + self._inode = inode + self._stat = None + self._lstat = None + self._path = None + + @property + def path(self): + if self._path is None: + self._path = join(self._scandir_path, self.name) + return self._path + + def stat(self, follow_symlinks=True): + if follow_symlinks: + if self._stat is None: + if self.is_symlink(): + self._stat = stat(self.path) + else: + if self._lstat is None: + self._lstat = lstat(self.path) + self._stat = self._lstat + return self._stat + else: + if self._lstat is None: + self._lstat = lstat(self.path) + return self._lstat + + def is_dir(self, follow_symlinks=True): + if (self._d_type == DT_UNKNOWN or + (follow_symlinks and self.is_symlink())): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False + return st.st_mode & 0o170000 == S_IFDIR + else: + return self._d_type == DT_DIR + + def is_file(self, follow_symlinks=True): + if (self._d_type == DT_UNKNOWN or + (follow_symlinks and self.is_symlink())): + try: + st = self.stat(follow_symlinks=follow_symlinks) + except OSError as e: + if e.errno != ENOENT: + raise + return False + return st.st_mode & 0o170000 == S_IFREG + else: + return self._d_type == DT_REG + + def is_symlink(self): + if self._d_type == DT_UNKNOWN: + try: + st = self.stat(follow_symlinks=False) + except OSError as e: + if e.errno != ENOENT: + raise + return False + return st.st_mode & 0o170000 == S_IFLNK + else: + return self._d_type == DT_LNK + + def inode(self): + return self._inode + + def __str__(self): + return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) + + __repr__ = __str__ + + def posix_error(filename): + errno = ctypes.get_errno() + exc = OSError(errno, strerror(errno)) + exc.filename = filename + return exc + + def scandir_python(path=unicode('.')): + """Like os.listdir(), but yield DirEntry objects instead of returning + a list of names. + """ + if isinstance(path, bytes): + opendir_path = path + is_bytes = True + else: + opendir_path = path.encode(file_system_encoding) + is_bytes = False + dir_p = opendir(opendir_path) + if not dir_p: + raise posix_error(path) + try: + result = Dirent_p() + while True: + entry = Dirent() + if readdir_r(dir_p, entry, result): + raise posix_error(path) + if not result: + break + name = entry.d_name + if name not in (b'.', b'..'): + if not is_bytes: + name = name.decode(file_system_encoding) + yield PosixDirEntry(path, name, entry.d_type, entry.d_ino) + finally: + if closedir(dir_p): + raise posix_error(path) + + if _scandir is not None: + scandir_c = _scandir.scandir + DirEntry_c = _scandir.DirEntry + + if _scandir is not None: + scandir = scandir_c + DirEntry = DirEntry_c + elif ctypes is not None and have_dirent_d_type: + scandir = scandir_python + DirEntry = PosixDirEntry + else: + scandir = scandir_generic + DirEntry = GenericDirEntry + + +# Some other system -- no d_type or stat information +else: + scandir = scandir_generic + DirEntry = GenericDirEntry + + +def _walk(top, topdown=True, onerror=None, followlinks=False): + """Like Python 3.5's implementation of os.walk() -- faster than + the pre-Python 3.5 version as it uses scandir() internally. + """ + dirs = [] + nondirs = [] + + # We may not have read permission for top, in which case we can't + # get a list of the files the directory contains. os.walk + # always suppressed the exception then, rather than blow up for a + # minor reason when (say) a thousand readable directories are still + # left to visit. That logic is copied here. + try: + scandir_it = scandir(top) + except OSError as error: + if onerror is not None: + onerror(error) + return + + while True: + try: + try: + entry = next(scandir_it) + except StopIteration: + break + except OSError as error: + if onerror is not None: + onerror(error) + return + + try: + is_dir = entry.is_dir() + except OSError: + # If is_dir() raises an OSError, consider that the entry is not + # a directory, same behaviour than os.path.isdir(). + is_dir = False + + if is_dir: + dirs.append(entry.name) + else: + nondirs.append(entry.name) + + if not topdown and is_dir: + # Bottom-up: recurse into sub-directory, but exclude symlinks to + # directories if followlinks is False + if followlinks: + walk_into = True + else: + try: + is_symlink = entry.is_symlink() + except OSError: + # If is_symlink() raises an OSError, consider that the + # entry is not a symbolic link, same behaviour than + # os.path.islink(). + is_symlink = False + walk_into = not is_symlink + + if walk_into: + for entry in walk(entry.path, topdown, onerror, followlinks): + yield entry + + # Yield before recursion if going top down + if topdown: + yield top, dirs, nondirs + + # Recurse into sub-directories + for name in dirs: + new_path = join(top, name) + # Issue #23605: os.path.islink() is used instead of caching + # entry.is_symlink() result during the loop on os.scandir() because + # the caller can replace the directory entry during the "yield" + # above. + if followlinks or not islink(new_path): + for entry in walk(new_path, topdown, onerror, followlinks): + yield entry + else: + # Yield after recursion if going bottom up + yield top, dirs, nondirs + + +if IS_PY3 or sys.platform != 'win32': + walk = _walk +else: + # Fix for broken unicode handling on Windows on Python 2.x, see: + # https://github.com/benhoyt/scandir/issues/54 + file_system_encoding = sys.getfilesystemencoding() + + def walk(top, topdown=True, onerror=None, followlinks=False): + if isinstance(top, bytes): + top = top.decode(file_system_encoding) + return _walk(top, topdown, onerror, followlinks) diff --git a/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_execfile.py b/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_execfile.py index c02f8ecf..75db0a82 100644 --- a/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_execfile.py +++ b/src/debugpy/_vendored/pydevd/_pydev_imps/_pydev_execfile.py @@ -1,4 +1,4 @@ -#We must redefine it in Py3k if it's not already there +# We must redefine it in Py3k if it's not already there def execfile(file, glob=None, loc=None): if glob is None: import sys @@ -14,12 +14,12 @@ def execfile(file, glob=None, loc=None): stream = tokenize.open(file) # @UndefinedVariable else: # version 3.0 or 3.1 - detect_encoding = tokenize.detect_encoding(open(file, mode="rb" ).readline) + detect_encoding = tokenize.detect_encoding(open(file, mode="rb").readline) stream = open(file, encoding=detect_encoding[0]) try: contents = stream.read() finally: stream.close() - #execute the script (note: it's important to compile first to have the filename set in debug mode) - exec(compile(contents+"\n", file, 'exec'), glob, loc) \ No newline at end of file + # execute the script (note: it's important to compile first to have the filename set in debug mode) + exec(compile(contents + "\n", file, 'exec'), glob, loc) diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py index 62909809..4bad7525 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_api.py @@ -258,12 +258,17 @@ class PyDevdAPI(object): elif thread_id.startswith('__frame__:'): sys.stderr.write("Can't set next statement in tasklet: %s\n" % (thread_id,)) - def request_reload_code(self, py_db, seq, module_name): + def request_reload_code(self, py_db, seq, module_name, filename): + ''' + :param seq: if -1 no message will be sent back when the reload is done. + + Note: either module_name or filename may be None (but not both at the same time). + ''' thread_id = '*' # Any thread # Note: not going for the main thread because in this case it'd only do the load # when we stopped on a breakpoint. py_db.post_method_as_internal_command( - thread_id, internal_reload_code, seq, module_name) + thread_id, internal_reload_code, seq, module_name, filename) def request_change_variable(self, py_db, seq, thread_id, frame_id, scope, attr, value): ''' @@ -1038,6 +1043,9 @@ class PyDevdAPI(object): py_db.terminate_requested = True run_as_pydevd_daemon_thread(py_db, self._terminate_if_commands_processed, py_db) + def setup_auto_reload_watcher(self, py_db, enable_auto_reload, watch_dirs, poll_target_time, exclude_patterns, include_patterns): + py_db.setup_auto_reload_watcher(enable_auto_reload, watch_dirs, poll_target_time, exclude_patterns, include_patterns) + def _list_ppid_and_pid(): _TH32CS_SNAPPROCESS = 0x00000002 diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py index 08d18386..4702ec3e 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_comm.py @@ -88,20 +88,21 @@ from _pydevd_bundle.pydevd_thread_lifecycle import pydevd_find_thread_by_id, res from _pydevd_bundle.pydevd_dont_trace_files import PYDEV_FILE import dis from _pydevd_bundle.pydevd_frame_utils import create_frames_list_from_exception_cause +import pydevd_file_utils try: from urllib import quote_plus, unquote_plus # @UnresolvedImport except: from urllib.parse import quote_plus, unquote_plus # @Reimport @UnresolvedImport import pydevconsole -from _pydevd_bundle import pydevd_vars, pydevd_utils, pydevd_io +from _pydevd_bundle import pydevd_vars, pydevd_utils, pydevd_io, pydevd_reload from _pydevd_bundle import pydevd_xml from _pydevd_bundle import pydevd_vm_type import sys import traceback from _pydevd_bundle.pydevd_utils import quote_smart as quote, compare_object_attrs_key, \ - notify_about_gevent_if_needed, isinstance_checked, ScopeRequest -from _pydev_bundle import pydev_log + notify_about_gevent_if_needed, isinstance_checked, ScopeRequest, getattr_checked +from _pydev_bundle import pydev_log, fsnotify from _pydev_bundle.pydev_log import exception as pydev_log_exception from _pydev_bundle import _pydev_completer @@ -310,6 +311,40 @@ class ReaderThread(PyDBDaemonThread): self.process_net_command(self.py_db, cmd_id, seq, text) +class FSNotifyThread(PyDBDaemonThread): + + def __init__(self, py_db, api, watch_dirs): + PyDBDaemonThread.__init__(self, py_db) + self.api = api + self.setName("pydevd.FSNotifyThread") + self.watcher = fsnotify.Watcher() + self.watch_dirs = watch_dirs + + @overrides(PyDBDaemonThread._on_run) + def _on_run(self): + try: + pydev_log.info('Watching directories for code reload:\n---\n%s\n---' % ('\n'.join(sorted(self.watch_dirs)))) + + # i.e.: The first call to set_tracked_paths will do a full scan, so, do it in the thread + # too (after everything is configured). + self.watcher.set_tracked_paths(self.watch_dirs) + while not self._kill_received: + for change_enum, change_path in self.watcher.iter_changes(): + # We're only interested in modified events + if change_enum == fsnotify.Change.modified: + pydev_log.info('Modified: %s', change_path) + self.api.request_reload_code(self.py_db, -1, None, change_path) + else: + pydev_log.info('Ignored (add or remove) change in: %s', change_path) + except: + pydev_log.exception('Error when waiting for filesystem changes in FSNotifyThread.') + + @overrides(PyDBDaemonThread.do_kill_pydev_thread) + def do_kill_pydev_thread(self): + self.watcher.dispose() + PyDBDaemonThread.do_kill_pydev_thread(self) + + class WriterThread(PyDBDaemonThread): ''' writer thread writes out the commands in an infinite loop ''' @@ -532,32 +567,67 @@ class InternalThreadCommandForAnyThread(InternalThreadCommand): InternalThreadCommand.do_it(self, dbg) -def internal_reload_code(dbg, seq, module_name): - module_name = module_name - if module_name not in sys.modules: - if '.' in module_name: - new_module_name = module_name.split('.')[-1] - if new_module_name in sys.modules: - module_name = new_module_name +def _send_io_message(py_db, s): + cmd = py_db.cmd_factory.make_io_message(s, 2) + if py_db.writer is not None: + py_db.writer.add_command(cmd) - reloaded_ok = False - if module_name not in sys.modules: - sys.stderr.write('pydev debugger: Unable to find module to reload: "' + module_name + '".\n') - # Too much info... - # sys.stderr.write('pydev debugger: This usually means you are trying to reload the __main__ module (which cannot be reloaded).\n') +def internal_reload_code(dbg, seq, module_name, filename): + try: + found_module_to_reload = False + if IS_PY2 and isinstance(filename, unicode): + filename = filename.encode(sys.getfilesystemencoding()) + + if module_name is not None: + module_name = module_name + if module_name not in sys.modules: + if '.' in module_name: + new_module_name = module_name.split('.')[-1] + if new_module_name in sys.modules: + module_name = new_module_name + + modules_to_reload = {} + module = sys.modules.get(module_name) + if module is not None: + modules_to_reload[id(module)] = (module, module_name) + + if filename: + filename = pydevd_file_utils.normcase(filename) + for module_name, module in sys.modules.copy().items(): + f = getattr_checked(module, '__file__') + if f is not None: + if f.endswith(('.pyc', '.pyo')): + f = f[:-1] + + if pydevd_file_utils.normcase(f) == filename: + modules_to_reload[id(module)] = (module, module_name) + + if not modules_to_reload: + if filename and module_name: + _send_io_message(dbg, 'code reload: Unable to find module %s to reload for path: %s\n' % (module_name, filename)) + elif filename: + _send_io_message(dbg, 'code reload: Unable to find module to reload for path: %s\n' % (filename,)) + elif module_name: + _send_io_message(dbg, 'code reload: Unable to find module to reload: %s\n' % (module_name,)) - else: - sys.stderr.write('pydev debugger: Start reloading module: "' + module_name + '" ... \n') - from _pydevd_bundle import pydevd_reload - if pydevd_reload.xreload(sys.modules[module_name]): - sys.stderr.write('pydev debugger: reload finished\n') - reloaded_ok = True else: - sys.stderr.write('pydev debugger: reload finished without applying any change\n') + # Too much info... + # _send_io_message(dbg, 'code reload: This usually means you are trying to reload the __main__ module (which cannot be reloaded).\n') + from _pydevd_bundle import pydevd_reload + for module, module_name in modules_to_reload.values(): + _send_io_message(dbg, 'code reload: Start reloading module: "' + module_name + '" ... \n') + found_module_to_reload = True - cmd = dbg.cmd_factory.make_reloaded_code_message(seq, reloaded_ok) - dbg.writer.add_command(cmd) + if pydevd_reload.xreload(module): + _send_io_message(dbg, 'code reload: reload finished\n') + else: + _send_io_message(dbg, 'code reload: reload finished without applying any change\n') + + cmd = dbg.cmd_factory.make_reloaded_code_message(seq, found_module_to_reload) + dbg.writer.add_command(cmd) + except: + pydev_log.exception('Error reloading code') class InternalGetThreadStack(InternalThreadCommand): diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_dont_trace_files.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_dont_trace_files.py index 2e1eb115..6cc66476 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_dont_trace_files.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_dont_trace_files.py @@ -148,6 +148,7 @@ DONT_TRACE = { 'pydevd_vars.py': PYDEV_FILE, 'pydevd_vm_type.py': PYDEV_FILE, 'pydevd_xml.py': PYDEV_FILE, + 'scandir_vendored.py': PYDEV_FILE, } if IS_PY3K: diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_json.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_json.py index a9e8fe29..387e3a8e 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_json.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_net_command_factory_json.py @@ -390,6 +390,10 @@ class NetCommandFactoryJson(NetCommandFactory): def make_thread_run_message(self, *args, **kwargs): return NULL_NET_COMMAND # Not a part of the debug adapter protocol + @overrides(NetCommandFactory.make_reloaded_code_message) + def make_reloaded_code_message(self, *args, **kwargs): + return NULL_NET_COMMAND # Not a part of the debug adapter protocol + @overrides(NetCommandFactory.make_input_requested_message) def make_input_requested_message(self, started): event = pydevd_schema.PydevdInputRequestedEvent(body={}) diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py index 98dd6b10..7ad5a552 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command.py @@ -170,8 +170,13 @@ class _PyDevCommandProcessor(object): cmd_smart_step_into = _cmd_set_next def cmd_reload_code(self, py_db, cmd_id, seq, text): - module_name = text.strip() - self.api.request_reload_code(py_db, seq, module_name) + text = text.strip() + if '\t' not in text: + module_name = text.strip() + filename = None + else: + module_name, filename = text.split('\t', 1) + self.api.request_reload_code(py_db, seq, module_name, filename) def cmd_change_variable(self, py_db, cmd_id, seq, text): # the text is: thread\tstackframe\tFRAME|GLOBAL\tattribute_to_change\tvalue_to_change diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command_json.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command_json.py index 22f8be00..53d3b11a 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command_json.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_process_net_command_json.py @@ -397,6 +397,51 @@ class PyDevJsonCommandProcessor(object): self.api.set_ignore_system_exit_codes(py_db, ignore_system_exit_codes) + auto_reload = args.get('autoReload', {}) + if not isinstance(auto_reload, dict): + pydev_log.info('Expected autoReload to be a dict. Received: %s' % (auto_reload,)) + auto_reload = {} + + enable_auto_reload = auto_reload.get('enable', False) + watch_dirs = auto_reload.get('watchDirectories') + if not watch_dirs: + watch_dirs = [] + # Note: by default this is no longer done because on some cases there are entries in the PYTHONPATH + # such as the home directory or /python/x64, where the site packages are in /python/x64/libs, so, + # we only watch the current working directory as well as executed script. + # check = getattr(sys, 'path', [])[:] + # # By default only watch directories that are in the project roots / + # # program dir (if available), sys.argv[0], as well as the current dir (we don't want to + # # listen to the whole site-packages by default as it can be huge). + # watch_dirs = [pydevd_file_utils.absolute_path(w) for w in check] + # watch_dirs = [w for w in watch_dirs if py_db.in_project_roots_filename_uncached(w) and os.path.isdir(w)] + + program = args.get('program') + if program: + if os.path.isdir(program): + watch_dirs.append(program) + else: + watch_dirs.append(os.path.dirname(program)) + watch_dirs.append(os.path.abspath('.')) + + argv = getattr(sys, 'argv', []) + if argv: + f = argv[0] + if os.path.isdir(f): + watch_dirs.append(program) + else: + watch_dirs.append(os.path.dirname(f)) + + if not isinstance(watch_dirs, (list, set, tuple)): + watch_dirs = (watch_dirs,) + watch_dirs = set(pydevd_file_utils.get_path_with_real_case(w) for w in watch_dirs) + + poll_target_time = auto_reload.get('pollingInterval', 1) + exclude_patterns = auto_reload.get('exclude', ('**/.git/**', '**/__pycache__/**', '**/node_modules/**', '**/.metadata/**', '**/site-packages/**')) + include_patterns = auto_reload.get('include', ('**/*.py', '**/*.pyw')) + self.api.setup_auto_reload_watcher( + py_db, enable_auto_reload, watch_dirs, poll_target_time, exclude_patterns, include_patterns) + if self._options.stop_on_entry and start_reason == 'launch': self.api.stop_on_entry() diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_reload.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_reload.py index 9ad32a8f..691d2ff8 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_reload.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_reload.py @@ -98,13 +98,11 @@ target namespace. """ -import imp -from _pydev_bundle.pydev_imports import Exec +from _pydev_bundle.pydev_imports import execfile from _pydevd_bundle import pydevd_dont_trace -import sys -import traceback import types from _pydev_bundle import pydev_log +from _pydevd_bundle.pydevd_constants import get_global_debugger NO_DEBUG = 0 LEVEL1 = 1 @@ -113,22 +111,18 @@ LEVEL2 = 2 DEBUG = NO_DEBUG -def write(*args): - new_lst = [] - for a in args: - new_lst.append(str(a)) - - msg = ' '.join(new_lst) - sys.stdout.write('%s\n' % (msg,)) - - def write_err(*args): - new_lst = [] - for a in args: - new_lst.append(str(a)) + py_db = get_global_debugger() + if py_db is not None: + new_lst = [] + for a in args: + new_lst.append(str(a)) - msg = ' '.join(new_lst) - sys.stderr.write('pydev debugger: %s\n' % (msg,)) + msg = ' '.join(new_lst) + s = 'code reload: %s\n' % (msg,) + cmd = py_db.cmd_factory.make_io_message(s, 2) + if py_db.writer is not None: + py_db.writer.add_command(cmd) def notify_info0(*args): @@ -137,12 +131,12 @@ def notify_info0(*args): def notify_info(*args): if DEBUG >= LEVEL1: - write(*args) + write_err(*args) def notify_info2(*args): if DEBUG >= LEVEL2: - write(*args) + write_err(*args) def notify_error(*args): @@ -197,51 +191,19 @@ def xreload(mod): #======================================================================================================================= class Reload: - def __init__(self, mod): + def __init__(self, mod, mod_name=None, mod_filename=None): self.mod = mod + self.mod_name = (mod_name or mod.__name__) if mod_name else None + self.mod_filename = (mod_filename or mod.__file__) if mod else None self.found_change = False def apply(self): mod = self.mod self._on_finish_callbacks = [] try: - # Get the module name, e.g. 'foo.bar.whatever' - modname = mod.__name__ # Get the module namespace (dict) early; this is part of the type check modns = mod.__dict__ - # Parse it into package name and module name, e.g. 'foo.bar' and 'whatever' - i = modname.rfind(".") - if i >= 0: - pkgname, modname = modname[:i], modname[i + 1:] - else: - pkgname = None - # Compute the search path - if pkgname: - # We're not reloading the package, only the module in it - pkg = sys.modules[pkgname] - path = pkg.__path__ # Search inside the package - else: - # Search the top-level module path - pkg = None - path = None # Make find_module() uses the default search path - # Find the module; may raise ImportError - (stream, filename, (suffix, mode, kind)) = imp.find_module(modname, path) - # Turn it into a code object - try: - # Is it Python source code or byte code read from a file? - if kind not in (imp.PY_COMPILED, imp.PY_SOURCE): - # Fall back to built-in reload() - notify_error('Could not find source to reload (mod: %s)' % (modname,)) - return - if kind == imp.PY_SOURCE: - source = stream.read() - code = compile(source, filename, "exec") - else: - import marshal - code = marshal.load(stream) - finally: - if stream: - stream.close() + # Execute the code. We copy the module dict to a temporary; then # clear the module dict; then execute the new code in the module # dict; then swap things back and around. This trick (due to @@ -250,8 +212,22 @@ class Reload: # object. new_namespace = modns.copy() new_namespace.clear() - new_namespace["__name__"] = modns["__name__"] - Exec(code, new_namespace) + if self.mod_filename: + new_namespace["__file__"] = self.mod_filename + try: + new_namespace["__builtins__"] = __builtins__ + except NameError: + raise # Ok if not there. + + if self.mod_name: + new_namespace["__name__"] = self.mod_name + if new_namespace["__name__"] == '__main__': + # We do this because usually the __main__ starts-up the program, guarded by + # the if __name__ == '__main__', but we don't want to start the program again + # on a reload. + new_namespace["__name__"] = '__main_reloaded__' + + execfile(self.mod_filename, new_namespace, new_namespace) # Now we get to the hard part oldnames = set(modns) newnames = set(new_namespace) @@ -308,7 +284,8 @@ class Reload: if type(oldobj) is not type(newobj): # Cop-out: if the type changed, give up - notify_error('Type of: %s changed... Skipping.' % (oldobj,)) + if name not in ('__builtins__',): + notify_error('Type of: %s (old: %s != new: %s) changed... Skipping.' % (name, type(oldobj), type(newobj))) return if isinstance(newobj, types.FunctionType): diff --git a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py index b6d27d56..91027c6d 100644 --- a/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py +++ b/src/debugpy/_vendored/pydevd/_pydevd_bundle/pydevd_utils.py @@ -297,6 +297,13 @@ def hasattr_checked(obj, name): else: return True +def getattr_checked(obj, name): + try: + return getattr(obj, name) + except: + # i.e.: Handle any exception, not only AttributeError. + return None + def dir_checked(obj): try: diff --git a/src/debugpy/_vendored/pydevd/pydevd.py b/src/debugpy/_vendored/pydevd/pydevd.py index 240dd6e1..d815bf3e 100644 --- a/src/debugpy/_vendored/pydevd/pydevd.py +++ b/src/debugpy/_vendored/pydevd/pydevd.py @@ -27,7 +27,7 @@ from _pydev_imps._pydev_saved_modules import thread from _pydev_imps._pydev_saved_modules import threading from _pydev_imps._pydev_saved_modules import time from _pydevd_bundle import pydevd_extension_utils, pydevd_frame_utils, pydevd_constants -from _pydevd_bundle.pydevd_filtering import FilesFiltering +from _pydevd_bundle.pydevd_filtering import FilesFiltering, glob_matches_path from _pydevd_bundle import pydevd_io, pydevd_vm_type from _pydevd_bundle import pydevd_utils from _pydev_bundle.pydev_console_utils import DebugConsoleStdIn @@ -62,7 +62,7 @@ from pydevd_file_utils import get_fullname, get_package_dir from os.path import abspath as os_path_abspath import pydevd_tracing from _pydevd_bundle.pydevd_comm import (InternalThreadCommand, InternalThreadCommandForAnyThread, - create_server_socket) + create_server_socket, FSNotifyThread) from _pydevd_bundle.pydevd_comm import(InternalConsoleExec, _queue, ReaderThread, GetGlobalDebugger, get_global_debugger, set_global_debugger, WriterThread, @@ -490,6 +490,7 @@ class PyDB(object): self.reader = None self.writer = None + self._fsnotify_thread = None self.created_pydb_daemon_threads = {} self._waiting_for_connection_thread = None self._on_configuration_done_event = threading.Event() @@ -537,6 +538,7 @@ class PyDB(object): self.ready_to_run = False self._main_lock = thread.allocate_lock() self._lock_running_thread_ids = thread.allocate_lock() + self._lock_create_fs_notify = thread.allocate_lock() self._py_db_command_thread_event = threading.Event() if set_as_global: CustomFramesContainer._py_db_command_thread_event = self._py_db_command_thread_event @@ -666,6 +668,77 @@ class PyDB(object): # Stop the tracing as the last thing before the actual shutdown for a clean exit. atexit.register(stoptrace) + def setup_auto_reload_watcher(self, enable_auto_reload, watch_dirs, poll_target_time, exclude_patterns, include_patterns): + try: + with self._lock_create_fs_notify: + + # When setting up, dispose of the previous one (if any). + if self._fsnotify_thread is not None: + self._fsnotify_thread.do_kill_pydev_thread() + self._fsnotify_thread = None + + if not enable_auto_reload: + return + + exclude_patterns = tuple(exclude_patterns) + include_patterns = tuple(include_patterns) + + def accept_directory(absolute_filename, cache={}): + try: + return cache[absolute_filename] + except: + if absolute_filename and absolute_filename[-1] not in ('/', '\\'): + # I.e.: for directories we always end with '/' or '\\' so that + # we match exclusions such as "**/node_modules/**" + absolute_filename += os.path.sep + + # First include what we want + for include_pattern in include_patterns: + if glob_matches_path(absolute_filename, include_pattern): + cache[absolute_filename] = True + return True + + # Then exclude what we don't want + for exclude_pattern in exclude_patterns: + if glob_matches_path(absolute_filename, exclude_pattern): + cache[absolute_filename] = False + return False + + # By default track all directories not excluded. + cache[absolute_filename] = True + return True + + def accept_file(absolute_filename, cache={}): + try: + return cache[absolute_filename] + except: + # First include what we want + for include_pattern in include_patterns: + if glob_matches_path(absolute_filename, include_pattern): + cache[absolute_filename] = True + return True + + # Then exclude what we don't want + for exclude_pattern in exclude_patterns: + if glob_matches_path(absolute_filename, exclude_pattern): + cache[absolute_filename] = False + return False + + # By default don't track files not included. + cache[absolute_filename] = False + return False + + self._fsnotify_thread = FSNotifyThread(self, PyDevdAPI(), watch_dirs) + watcher = self._fsnotify_thread.watcher + watcher.accept_directory = accept_directory + watcher.accept_file = accept_file + + watcher.target_time_for_single_scan = poll_target_time + watcher.target_time_for_notification = poll_target_time + self._fsnotify_thread.start() + except: + pydev_log.exception('Error setting up auto-reload.') + def get_arg_ppid(self): try: setup = SetupHolder.setup @@ -1077,6 +1150,9 @@ class PyDB(object): return cache[cache_key] + def in_project_roots_filename_uncached(self, absolute_filename): + return self._files_filtering.in_project_roots(absolute_filename) + def _clear_filters_caches(self): self._in_project_scope_cache.clear() self._exclude_by_filter_cache.clear() diff --git a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py index e6b7fd89..34f29082 100644 --- a/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py +++ b/src/debugpy/_vendored/pydevd/tests_python/test_debugger.py @@ -749,7 +749,7 @@ def test_case_16_resolve_numpy_array(case_setup): ''.format(builtin_qualifier), ], '