diff --git a/ipykernel/debugger.py b/ipykernel/debugger.py index dfe5d0a9b..daf0eca3c 100644 --- a/ipykernel/debugger.py +++ b/ipykernel/debugger.py @@ -1,7 +1,7 @@ +from pathlib import Path import sys import os import re -import threading import zmq from zmq.utils import jsonapi @@ -349,7 +349,7 @@ def _accept_stopped_thread(self, thread_name): 'Thread-4' ] return thread_name not in forbid_list - + async def handle_stopped_event(self): # Wait for a stopped event message in the stopped queue # This message is used for triggering the 'threads' request @@ -375,8 +375,14 @@ def start(self): if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) host, port = self.debugpy_client.get_host_port() - code = 'import debugpy;' - code += 'debugpy.listen(("' + host + '",' + port + '))' + code = "import debugpy\n" + # Write debugpy logs? + #code += f'import debugpy; debugpy.log_to({str(Path(__file__).parent)!r});' + code += 'debugpy.listen(("' + host + '",' + port + '))\n' + code += (Path(__file__).parent / "filtered_pydb.py").read_text("utf8") + # Write pydevd logs? + # code += f'\npydevd.DebugInfoHolder.PYDEVD_DEBUG_FILE = {str(Path(__file__).parent / "debugpy.pydev.log")!r}\n' + # code += "pydevd.DebugInfoHolder.DEBUG_TRACE_LEVEL = 2\n" content = { 'code': code, 'silent': True @@ -449,29 +455,7 @@ async def source(self, message): return reply async def stackTrace(self, message): - reply = await self._forward_message(message) - # The stackFrames array can have the following content: - # { frames from the notebook} - # ... - # { 'id': xxx, 'name': '', ... } <= this is the first frame of the code from the notebook - # { frames from ipykernel } - # ... - # {'id': yyy, 'name': '', ... } <= this is the first frame of ipykernel code - # or only the frames from the notebook. - # We want to remove all the frames from ipykernel when they are present. - try: - sf_list = reply["body"]["stackFrames"] - module_idx = len(sf_list) - next( - i - for i, v in enumerate(reversed(sf_list), 1) - if v["name"] == "" and i != 1 - ) - reply["body"]["stackFrames"] = reply["body"]["stackFrames"][ - : module_idx + 1 - ] - except StopIteration: - pass - return reply + return await self._forward_message(message) def accept_variable(self, variable_name): forbid_list = [ @@ -524,8 +508,11 @@ async def attach(self, message): # The ipykernel source is in the call stack, so the user # has to manipulate the step-over and step-into in a wize way. # Set debugOptions for breakpoints in python standard library source. - if not self.just_my_code: - message['arguments']['debugOptions'] = [ 'DebugStdLib' ] + message['arguments']['options'] = f'DEBUG_STDLIB={not self.just_my_code}' + # Explicitly ignore IPython implicit hooks ? + message['arguments']['rules'] = [ + # { "module": "IPython.core.displayhook", "include": False }, + ] return await self._forward_message(message) async def configurationDone(self, message): @@ -567,7 +554,7 @@ async def debugInfo(self, message): async def inspectVariables(self, message): self.variable_explorer.untrack_all() # looks like the implementation of untrack_all in ptvsd - # destroys objects we nee din track. We have no choice but + # destroys objects we need in track. We have no choice but # reinstantiate the object self.variable_explorer = VariableExplorer() self.variable_explorer.track() diff --git a/ipykernel/filtered_pydb.py b/ipykernel/filtered_pydb.py new file mode 100644 index 000000000..d29508d52 --- /dev/null +++ b/ipykernel/filtered_pydb.py @@ -0,0 +1,76 @@ +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import pydevd + +__db = pydevd.get_global_debugger() +if __db: + __original = __db.get_file_type + __initial = True + def get_file_type(self, frame, abs_real_path_and_basename=None, _cache_file_type=pydevd._CACHE_FILE_TYPE): + ''' + :param abs_real_path_and_basename: + The result from get_abs_path_real_path_and_base_from_file or + get_abs_path_real_path_and_base_from_frame. + + :return + _pydevd_bundle.pydevd_dont_trace_files.PYDEV_FILE: + If it's a file internal to the debugger which shouldn't be + traced nor shown to the user. + + _pydevd_bundle.pydevd_dont_trace_files.LIB_FILE: + If it's a file in a library which shouldn't be traced. + + None: + If it's a regular user file which should be traced. + ''' + global __initial + if __initial: + __initial = False + _cache_file_type.clear() + # Copied normalization: + if abs_real_path_and_basename is None: + try: + # Make fast path faster! + abs_real_path_and_basename = pydevd.NORM_PATHS_AND_BASE_CONTAINER[frame.f_code.co_filename] + except: + abs_real_path_and_basename = pydevd.get_abs_path_real_path_and_base_from_frame(frame) + + cache_key = (frame.f_code.co_firstlineno, abs_real_path_and_basename[0], frame.f_code) + try: + return _cache_file_type[cache_key] + except KeyError: + pass + + ret = __original(frame, abs_real_path_and_basename, _cache_file_type) + if ret is self.PYDEV_FILE: + return ret + if not hasattr(frame, "f_locals"): + return ret + + # if either user or lib, check with our logic + # (we check "user" code in case any of the libs we use are in edit install) + # logic outline: + # - check if current frame is IPython bottom frame (if so filter it) + # - if not, check all ancestor for ipython bottom. Filter if not present. + # - if debugging / developing, do some sanity check of ignored frames, and log any unexecpted frames + + # do not cache, these frames might show up on different sides of the bottom frame! + del _cache_file_type[cache_key] + if frame.f_locals.get("__tracebackhide__") == "__ipython_bottom__": + # Current frame is bottom frame, hide it! + pydevd.pydev_log.debug("Ignoring IPython bottom frame: %s - %s", frame.f_code.co_filename, frame.f_code.co_name) + ret = _cache_file_type[cache_key] = self.PYDEV_FILE + else: + f = frame + while f is not None: + if f.f_locals.get("__tracebackhide__") == "__ipython_bottom__": + # we found ipython bottom in stack, do not change type + return ret + f = f.f_back + pydevd.pydev_log.debug("Ignoring ipykernel frame: %s - %s", frame.f_code.co_filename, frame.f_code.co_name) + + ret = self.PYDEV_FILE + return ret + __db.get_file_type = get_file_type.__get__(__db, pydevd.PyDB) + __db.is_files_filter_enabled = True diff --git a/ipykernel/kernelbase.py b/ipykernel/kernelbase.py index c775398fc..270df5774 100644 --- a/ipykernel/kernelbase.py +++ b/ipykernel/kernelbase.py @@ -133,7 +133,7 @@ def _default_ident(self): # Experimental option to break in non-user code. # The ipykernel source is in the call stack, so the user # has to manipulate the step-over and step-into in a wize way. - debug_just_my_code = Bool(True, + debug_just_my_code = Bool(os.environ.get("IPYKERNEL_DEBUG_JUST_MY_CODE", "True").lower() == "true", help="""Set to False if you want to debug python standard and dependent libraries. """ ).tag(config=True) diff --git a/ipykernel/tests/test_debugger.py b/ipykernel/tests/test_debugger.py index 43a96ef22..8751f5eb3 100644 --- a/ipykernel/tests/test_debugger.py +++ b/ipykernel/tests/test_debugger.py @@ -1,3 +1,4 @@ +from queue import Empty import sys import pytest @@ -31,9 +32,32 @@ def wait_for_debug_request(kernel, command, arguments=None, full_reply=False): return reply if full_reply else reply["content"] +def wait_for_debug_event(kernel, event, timeout=TIMEOUT, verbose=False, full_reply=False): + msg = {"msg_type": "", "content": {}} + while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != event: + msg = kernel.get_iopub_msg(timeout=timeout) + if verbose: + print(msg.get("msg_type")) + if (msg.get("msg_type") == "debug_event"): + print(f' {msg["content"].get("event")}') + return msg if full_reply else msg["content"] + + +def assert_stack_names(kernel, expected_names, thread_id=1): + reply = wait_for_debug_request(kernel, "stackTrace", {"threadId": thread_id}) + names = [f.get("name") for f in reply["body"]["stackFrames"]] + # "" will be the name of the cell + assert names == expected_names + + @pytest.fixture -def kernel(): - with new_kernel() as kc: +def kernel(request): + if sys.platform == "win32": + import asyncio + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + argv = getattr(request, "param", []) + #argv.append("--log-level=DEBUG") + with new_kernel(argv) as kc: yield kc @@ -144,7 +168,7 @@ def test_stop_on_breakpoint(kernel_with_debug): kernel_with_debug, "setBreakpoints", { - "breakpoints": [{"line": 2}], + "breakpoints": [{"line": 2}, {"line": 5}], "source": {"path": source}, "sourceModified": False, }, @@ -153,16 +177,27 @@ def test_stop_on_breakpoint(kernel_with_debug): wait_for_debug_request(kernel_with_debug, "configurationDone", full_reply=True) kernel_with_debug.execute(code) - + # Wait for stop on breakpoint - msg = {"msg_type": "", "content": {}} - while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != "stopped": - msg = kernel_with_debug.get_iopub_msg(timeout=TIMEOUT) + msg = wait_for_debug_event(kernel_with_debug, "stopped") + assert msg["body"]["reason"] == "breakpoint" + assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) - assert msg["content"]["body"]["reason"] == "breakpoint" + wait_for_debug_request(kernel_with_debug, "continue", {"threadId": msg["body"].get("threadId", 1)}) + + # Wait for stop on breakpoint + msg = wait_for_debug_event(kernel_with_debug, "stopped") + assert msg["body"]["reason"] == "breakpoint" + stacks = wait_for_debug_request( + kernel_with_debug, + "stackTrace", + {"threadId": r["body"].get("threadId", 1)} + )["body"]["stackFrames"] + names = [f.get("name") for f in stacks] + assert stacks[0]["line"] == 2 + assert names == ["f", ""] -@pytest.mark.skipif(sys.version_info >= (3, 10), reason="TODO Does not work on Python 3.10") def test_breakpoint_in_cell_with_leading_empty_lines(kernel_with_debug): code = """ def f(a, b): @@ -189,13 +224,10 @@ def f(a, b): wait_for_debug_request(kernel_with_debug, "configurationDone", full_reply=True) kernel_with_debug.execute(code) - - # Wait for stop on breakpoint - msg = {"msg_type": "", "content": {}} - while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != "stopped": - msg = kernel_with_debug.get_iopub_msg(timeout=TIMEOUT) - assert msg["content"]["body"]["reason"] == "breakpoint" + # Wait for stop on breakpoint + msg = wait_for_debug_event(kernel_with_debug, "stopped") + assert msg["body"]["reason"] == "breakpoint" def test_rich_inspect_not_at_breakpoint(kernel_with_debug): @@ -220,6 +252,99 @@ def test_rich_inspect_not_at_breakpoint(kernel_with_debug): assert reply["body"]["data"] == {"text/plain": f"'{value}'"} +@pytest.mark.parametrize("kernel", [["--Kernel.debug_just_my_code=False"]], indirect=True) +def test_step_into_lib(kernel_with_debug): + code = """import traitlets +traitlets.validate('foo', 'bar') +""" + + r = wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code}) + source = r["body"]["sourcePath"] + + wait_for_debug_request( + kernel_with_debug, + "setBreakpoints", + { + "breakpoints": [{"line": 1}], + "source": {"path": source}, + "sourceModified": False, + }, + ) + + wait_for_debug_request(kernel_with_debug, "debugInfo") + + wait_for_debug_request(kernel_with_debug, "configurationDone") + kernel_with_debug.execute(code) + + # Wait for stop on breakpoint + r = wait_for_debug_event(kernel_with_debug, "stopped") + assert r["body"]["reason"] == "breakpoint" + assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) + + # Step over the import statement + wait_for_debug_request(kernel_with_debug, "next", {"threadId": r["body"].get("threadId", 1)}) + r = wait_for_debug_event(kernel_with_debug, "stopped") + assert r["body"]["reason"] == "step" + assert_stack_names(kernel_with_debug, [""], r["body"].get("threadId", 1)) + + # Attempt to step into the function call + wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": r["body"].get("threadId", 1)}) + r = wait_for_debug_event(kernel_with_debug, "stopped") + assert r["body"]["reason"] == "step" + assert_stack_names(kernel_with_debug, ["validate", ""], r["body"].get("threadId", 1)) + + +# Test with both lib code and only "my code" +@pytest.mark.parametrize("kernel", [[], ["--Kernel.debug_just_my_code=False"]], indirect=True) +def test_step_into_end(kernel_with_debug): + code = 'a = 5 + 5' + + r = wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code}) + source = r["body"]["sourcePath"] + + wait_for_debug_request( + kernel_with_debug, + "setBreakpoints", + { + "breakpoints": [{"line": 1}], + "source": {"path": source}, + "sourceModified": False, + }, + ) + + wait_for_debug_request(kernel_with_debug, "debugInfo") + + wait_for_debug_request(kernel_with_debug, "configurationDone") + kernel_with_debug.execute(code) + + # Wait for stop on breakpoint + r = wait_for_debug_event(kernel_with_debug, "stopped") + + # Attempt to step into the statement (will continue execution, but + # should stop on first line of next execute request) + wait_for_debug_request(kernel_with_debug, "stepIn", {"threadId": r["body"].get("threadId", 1)}) + # assert no stop statement is given + try: + r = wait_for_debug_event(kernel_with_debug, "stopped", timeout=3) + except Empty: + pass + else: + # we're stopped somewhere. Fail with trace + reply = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": r["body"].get("threadId", 1)}) + entries = [] + for f in reversed(reply["body"]["stackFrames"]): + source = f.get("source", {}).get("path") or "" + loc = f'{source} ({f.get("line")},{f.get("column")})' + entries.append(f'{loc}: {f.get("name")}') + raise AssertionError('Unexpectedly stopped. Debugger stack:\n {0}'.format("\n ".join(entries))) + + # execute some new code without breakpoints, assert it stops + code = 'print("bar")\nprint("alice")\n' + wait_for_debug_request(kernel_with_debug, "dumpCell", {"code": code}) + kernel_with_debug.execute(code) + wait_for_debug_event(kernel_with_debug, "stopped") + + def test_rich_inspect_at_breakpoint(kernel_with_debug): code = """def f(a, b): c = a + b @@ -248,11 +373,9 @@ def test_rich_inspect_at_breakpoint(kernel_with_debug): kernel_with_debug.execute(code) # Wait for stop on breakpoint - msg = {"msg_type": "", "content": {}} - while msg.get('msg_type') != 'debug_event' or msg["content"].get("event") != "stopped": - msg = kernel_with_debug.get_iopub_msg(timeout=TIMEOUT) + r = wait_for_debug_event(kernel_with_debug, "stopped") - stacks = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": 1})[ + stacks = wait_for_debug_request(kernel_with_debug, "stackTrace", {"threadId": r["body"].get("threadId", 1)})[ "body" ]["stackFrames"]