First proto: use `greenback` for sync func breakpointing

This works now for supporting a new `tractor.pause_from_sync()`
`tractor`-aware-replacement for `Pdb.set_trace()` from sync functions
which are also scheduled from our runtime. Uses `greenback` to do all
the magic of scheduling the bg `tractor._debug._pause()` task and
engaging the normal TTY locking machinery triggered by `await
tractor.breakpoint()`

Further this starts some public API renaming, making a switch to
`tractor.pause()` from `.breakpoint()` which IMO much better expresses
the semantics of the runtime intervention required to suffice
multi-process "breakpointing"; it also is an alternate name for the same
in computer science more generally: https://en.wikipedia.org/wiki/Breakpoint
It also avoids using the same name as the `breakpoint()` built-in which
is important since there **is alot more going on** when you call our
equivalent API.

Deats of that:
- add deprecation warning for `tractor.breakpoint()`
- add `tractor.pause()` and a shorthand, easier-to-type, alias `.pp()`
  for "pause-point" B)
- add `pause_from_sync()` as the new `breakpoint()`-from-sync-function
  hack which does all the `greenback` stuff for the user.

Still TODO:
- figure out where in the runtime and when to call
  `greenback.ensure_portal()`.
- fix the frame selection issue where
  `trio._core._ki._ki_protection_decorator:wrapper` seems to be always
  shown on REPL start as the selected frame..

tractor/__init__.py

@@ -48,6 +48,9 @@ from ._exceptions import (
 )
 from ._debug import (
     breakpoint,
+    pause,
+    pp,
+    pause_from_sync,
     post_mortem,
 )
 from . import msg
@@ -61,12 +64,12 @@ from ._runtime import Actor
 
 __all__ = [
     'Actor',
+    'BaseExceptionGroup',
     'Channel',
     'Context',
     'ContextCancelled',
     'ModuleNotExposed',
     'MsgStream',
-    'BaseExceptionGroup',
     'Portal',
     'RemoteActorError',
     'breakpoint',
@@ -79,7 +82,10 @@ __all__ = [
     'open_actor_cluster',
     'open_nursery',
     'open_root_actor',
+    'pause',
     'post_mortem',
+    'pp',
+    'pause_from_sync'
     'query_actor',
     'run_daemon',
     'stream',

tractor/_debug.py

@@ -374,7 +374,7 @@ async def wait_for_parent_stdin_hijack(
 
     This function is used by any sub-actor to acquire mutex access to
     the ``pdb`` REPL and thus the root's TTY for interactive debugging
-    (see below inside ``_breakpoint()``). It can be used to ensure that
+    (see below inside ``_pause()``). It can be used to ensure that
     an intermediate nursery-owning actor does not clobber its children
     if they are in debug (see below inside
     ``maybe_wait_for_debugger()``).
@@ -440,17 +440,29 @@ def mk_mpdb() -> tuple[MultiActorPdb, Callable]:
     return pdb, Lock.unshield_sigint
 
 
-async def _breakpoint(
+async def _pause(
 
-    debug_func,
+    debug_func: Callable | None = None,
+    release_lock_signal: trio.Event | None = None,
 
     # TODO:
     # shield: bool = False
+    task_status: TaskStatus[trio.Event] = trio.TASK_STATUS_IGNORED
 
 ) -> None:
     '''
-    Breakpoint entry for engaging debugger instance sync-interaction,
+    A pause point (more commonly known as a "breakpoint") interrupt
-    from async code, executing in actor runtime (task).
+    instruction for engaging a blocking debugger instance to
+    conduct manual console-based-REPL-interaction from within
+    `tractor`'s async runtime, normally from some single-threaded
+    and currently executing actor-hosted-`trio`-task in some
+    (remote) process.
+
+    NOTE: we use the semantics "pause" since it better encompasses
+    the entirety of the necessary global-runtime-state-mutation any
+    actor-task must access and lock in order to get full isolated
+    control over the process tree's root TTY:
+    https://en.wikipedia.org/wiki/Breakpoint
 
     '''
     __tracebackhide__ = True
@@ -559,6 +571,17 @@ async def _breakpoint(
         Lock.repl = pdb
 
     try:
+        # breakpoint()
+        if debug_func is None:
+            assert release_lock_signal, (
+                'Must pass `release_lock_signal: trio.Event` if no '
+                'trace func provided!'
+            )
+            print(f"{actor.uid} ENTERING WAIT")
+            task_status.started()
+            await release_lock_signal.wait()
+
+        else:
             # block here one (at the appropriate frame *up*) where
             # ``breakpoint()`` was awaited and begin handling stdio.
             log.debug("Entering the synchronous world of pdb")
@@ -708,8 +731,8 @@ def shield_sigint_handler(
         # elif debug_mode():
 
     else:  # XXX: shouldn't ever get here?
-        print("WTFWTFWTF")
+        raise RuntimeError("WTFWTFWTF")
-        raise KeyboardInterrupt
+        # raise KeyboardInterrupt("WTFWTFWTF")
 
     # NOTE: currently (at least on ``fancycompleter`` 0.9.2)
     # it looks to be that the last command that was run (eg. ll)
@@ -737,21 +760,18 @@ def shield_sigint_handler(
         # https://github.com/goodboy/tractor/issues/130#issuecomment-663752040
         # https://github.com/prompt-toolkit/python-prompt-toolkit/blob/c2c6af8a0308f9e5d7c0e28cb8a02963fe0ce07a/prompt_toolkit/patch_stdout.py
 
-        # XXX LEGACY: lol, see ``pdbpp`` issue:
-        # https://github.com/pdbpp/pdbpp/issues/496
-
 
 def _set_trace(
     actor: tractor.Actor | None = None,
     pdb: MultiActorPdb | None = None,
 ):
     __tracebackhide__ = True
-    actor = actor or tractor.current_actor()
+    actor: tractor.Actor = actor or tractor.current_actor()
 
     # start 2 levels up in user code
-    frame: Optional[FrameType] = sys._getframe()
+    frame: FrameType | None = sys._getframe()
     if frame:
-        frame = frame.f_back  # type: ignore
+        frame: FrameType = frame.f_back  # type: ignore
 
     if (
         frame
@@ -773,10 +793,66 @@ def _set_trace(
     pdb.set_trace(frame=frame)
 
 
-breakpoint = partial(
+# TODO: allow pausing from sync code, normally by remapping
-    _breakpoint,
+# python's builtin breakpoint() hook to this runtime aware version.
+def pause_from_sync() -> None:
+    import greenback
+
+    actor: tractor.Actor = tractor.current_actor()
+    task_can_release_tty_lock = trio.Event()
+
+    # spawn bg task which will lock out the TTY, we poll
+    # just below until the release event is reporting that task as
+    # waiting.. not the most ideal but works for now ;)
+    greenback.await_(
+        actor._service_n.start(partial(
+            _pause,
+            debug_func=None,
+            release_lock_signal=task_can_release_tty_lock,
+        ))
+    )
+    print("ENTER SYNC PAUSE")
+    pdb, undo_sigint = mk_mpdb()
+    try:
+        print("ENTER SYNC PAUSE")
+        # _set_trace(actor=actor)
+
+        # we entered the global ``breakpoint()`` built-in from sync
+        # code?
+        Lock.local_task_in_debug = 'sync'
+        frame: FrameType | None = sys._getframe()
+        print(f'FRAME: {str(frame)}')
+
+        frame: FrameType = frame.f_back  # type: ignore
+        print(f'FRAME: {str(frame)}')
+
+        frame: FrameType = frame.f_back  # type: ignore
+        print(f'FRAME: {str(frame)}')
+
+        pdb.set_trace(frame=frame)
+        # pdb.do_frame(
+        #     pdb.curindex
+
+    finally:
+        task_can_release_tty_lock.set()
+        undo_sigint()
+
+# using the "pause" semantics instead since
+# that better covers actually somewhat "pausing the runtime"
+# for this particular paralell task to do debugging B)
+pause = partial(
+    _pause,
     _set_trace,
 )
+pp = pause  # short-hand for "pause point"
+
+
+async def breakpoint(**kwargs):
+    log.warning(
+        '`tractor.breakpoint()` is deprecated!\n'
+        'Please use `tractor.pause()` instead!\n'
+    )
+    await pause(**kwargs)
 
 
 def _post_mortem(
@@ -801,7 +877,7 @@ def _post_mortem(
 
 
 post_mortem = partial(
-    _breakpoint,
+    _pause,
     _post_mortem,
 )
 

tractor/_runtime.py

@@ -95,6 +95,10 @@ async def _invoke(
     treat_as_gen: bool = False
     failed_resp: bool = False
 
+    if _state.debug_mode():
+        import greenback
+        await greenback.ensure_portal()
+
     # possibly a traceback (not sure what typing is for this..)
     tb = None
 
@@ -1862,4 +1866,6 @@ class Arbiter(Actor):
 
     ) -> None:
         uid = (str(uid[0]), str(uid[1]))
-        self._registry.pop(uid)
+        entry: tuple = self._registry.pop(uid, None)
+        if entry is None:
+            log.warning(f'Request to de-register {uid} failed?')