Drop diff state tracking in formatter

This was a major cause of error (particularly trying to get epoch
indexing working) and really isn't necessary; instead just have
`.diff()` always read from the underlying source array for current
index-step diffing and append/prepend slice construction.

Allows us to,
- drop `._last_read` state management and thus usage.
- better handle startup indexing by setting `.xy_nd_start/stop` to
  `None` initially so that the first update can be done in one large
  prepend.
- better understand and document the step curve "slice back to previous
  level" logic which is now heavily commented B)
- drop all the `slice_to_head` stuff from and instead allow each
  formatter to choose it's 1d segmenting.

piker/data/_pathops.py

@@ -64,18 +64,6 @@ class IncrementalFormatter(msgspec.Struct):
     viz: Viz
     index_field: str = 'index'
 
-    # last read from shm (usually due to an update call)
-    _last_read: tuple[
-        int,
-        int,
-        np.ndarray
-
-    ]
-
-    @property
-    def last_read(self) -> tuple | None:
-        return self._last_read
-
     # Incrementally updated xy ndarray formatted data, a pre-1d
     # format which is updated and cached independently of the final
     # pre-graphics-path 1d format.
@@ -99,8 +87,8 @@ class IncrementalFormatter(msgspec.Struct):
     # indexes which slice into the above arrays (which are allocated
     # based on source data shm input size) and allow retrieving
     # incrementally updated data.
-    xy_nd_start: int = 0
+    xy_nd_start: int | None = None
-    xy_nd_stop: int = 0
+    xy_nd_stop: int | None = None
 
     # TODO: eventually incrementally update 1d-pre-graphics path data?
     # x_1d: Optional[np.ndarray] = None
@@ -144,17 +132,14 @@ class IncrementalFormatter(msgspec.Struct):
         np.ndarray,
         np.ndarray,
     ]:
-        (
-            last_xfirst,
-            last_xlast,
-            last_array,
-            last_ivl,
-            last_ivr,
-            last_in_view,
-        ) = self.last_read
 
-        # TODO: can the renderer just call ``Viz.read()`` directly?
+        # TODO:
-        # unpack latest source data read
+        # - can the renderer just call ``Viz.read()`` directly? unpack
+        #   latest source data read
+        # - eventually maybe we can implement some kind of
+        #   transform on the ``QPainterPath`` that will more or less
+        #   detect the diff in "elements" terms? update diff state since
+        #   we've now rendered paths.
         (
             xfirst,
             xlast,
@@ -164,25 +149,42 @@ class IncrementalFormatter(msgspec.Struct):
             in_view,
         ) = new_read
 
+        index = array['index']
+
+        # if the first index in the read array is 0 then
+        # it means the source buffer has bee completely backfilled to
+        # available space.
+        src_start = index[0]
+        src_stop = index[-1] + 1
+
+        # these are the "formatted output data" indices
+        # for the pre-graphics arrays.
+        nd_start = self.xy_nd_start
+        nd_stop = self.xy_nd_stop
+
+        if (
+            nd_start is None
+        ):
+            assert nd_stop is None
+
+            # setup to do a prepend of all existing src history
+            nd_start = self.xy_nd_start = src_stop
+            # set us in a zero-to-append state
+            nd_stop = self.xy_nd_stop = src_stop
+
         # compute the length diffs between the first/last index entry in
         # the input data and the last indexes we have on record from the
         # last time we updated the curve index.
-        prepend_length = int(last_xfirst - xfirst)
+        prepend_length = int(nd_start - src_start)
-        append_length = int(xlast - last_xlast)
+        append_length = int(src_stop - nd_stop)
-
-        # if (
-        #     prepend_length < 0
-        #     or append_length < 0
-        # ):
-        #     breakpoint()
 
         # blah blah blah
         # do diffing for prepend, append and last entry
         return (
-            slice(xfirst, last_xfirst),
+            slice(src_start, nd_start),
             prepend_length,
             append_length,
-            slice(last_xlast, xlast),
+            slice(nd_stop, src_stop),
         )
 
     def _track_inview_range(
@@ -233,7 +235,6 @@ class IncrementalFormatter(msgspec.Struct):
         array_key: str,
         profiler: Profiler,
 
-        slice_to_head: int = -1,
         read_src_from_key: bool = True,
         slice_to_inview: bool = True,
 
@@ -311,12 +312,6 @@ class IncrementalFormatter(msgspec.Struct):
             self.xy_nd_stop = shm._last.value
             profiler('appened xy history: {append_length}')
 
-        # TODO: eventually maybe we can implement some kind of
-        # transform on the ``QPainterPath`` that will more or less
-        # detect the diff in "elements" terms?
-        # update diff state since we've now rendered paths.
-        self._last_read = new_read
-
         view_changed: bool = False
         view_range: tuple[int, int] = (ivl, ivr)
         if slice_to_inview:
@@ -365,7 +360,7 @@ class IncrementalFormatter(msgspec.Struct):
 
         # update the last "in view data range"
         if len(x_1d):
-            self._last_ivdr = x_1d[0], x_1d[slice_to_head]
+            self._last_ivdr = x_1d[0], x_1d[-1]
             if (x_1d[-1] == 0.5).any():
                 breakpoint()
 
@@ -760,41 +755,62 @@ class StepCurveFmtr(IncrementalFormatter):
         np.ndarray,
         slice,
     ]:
-        # for a step curve we slice from one datum prior
+        # NOTE: for a step curve we slice from one datum prior
         # to the current "update slice" to get the previous
         # "level".
-        last_2 = slice(
+        #
-            read_slc.start,
+        # why this is needed,
-            read_slc.stop+1,
+        # - the current new append slice will often have a zero
+        #   value in the latest datum-step (at least for zero-on-new
+        #   cases like vlm in the) as per configuration of the FSP
+        #   engine.
+        # - we need to look back a datum to get the last level which
+        #   will be used to terminate/complete the last step x-width
+        #   which will be set to pair with the last x-index THIS MEANS
+        #
+        # XXX: this means WE CAN'T USE the append slice since we need to
+        # "look backward" one step to get the needed back-to-zero level
+        # and the update data in ``new_from_src`` will only contain the
+        # latest new data.
+        back_1 = slice(
+            read_slc.start - 1,
+            read_slc.stop,
         )
-        y_nd_new = self.y_nd[last_2]
-        y_nd_new[:] = src_shm._array[last_2][array_key][:, None]
 
-        # NOTE: we can't use the append slice since we need to "look
+        to_write = src_shm._array[back_1]
-        # forward" one step to get the current level and copy it as
+        y_nd_new = self.y_nd[back_1]
-        # well? (though i still don't really grok why..)
+        y_nd_new[:] = to_write[array_key][:, None]
-        # y_nd_new[:] = new_from_src[array_key][:, None]
 
-        # XXX: old approach now duplicated above (we can probably drop
+        x_nd_new = self.x_nd[read_slc]
-        # this since the key part was the ``nd_stop + 1``
+        x_nd_new[:] = (
-        # if is_append:
+            new_from_src[self.index_field][:, None]
-        #     start = max(nd_stop - 1, 0)
+            +
-        #     end = src_shm._last.value
+            np.array([-0.5, 0.5])
-        #     y_nd_new = src_shm._array[start:end][array_key]#[:, np.newaxis]
+        )
-        #     slc = slice(start, end)
+
-        #     self.y_nd[slc] = np.broadcast_to(
+        # XXX: uncomment for debugging
-        #         y_nd_new[:, None],
+        # x_nd = self.x_nd[self.xy_slice]
-        #         (y_nd_new.size, 2),
+        # y_nd = self.y_nd[self.xy_slice]
+        # name = self.viz.name
+        # if 'dolla_vlm' in name:
+        #     s = 4
+        #     print(
+        #         f'{name}:\n'
+        #         'NEW_FROM_SRC:\n'
+        #         f'new_from_src: {new_from_src}\n\n'
+
+        #         f'PRE self.x_nd:'
+        #         f'\n{x_nd[-s:]}\n'
+        #         f'PRE self.y_nd:\n'
+        #         f'{y_nd[-s:]}\n\n'
+
+        #         f'TO WRITE:\n'
+        #         f'x_nd_new:\n'
+        #         f'{x_nd_new}\n'
+        #         f'y_nd_new:\n'
+        #         f'{y_nd_new}\n'
         #     )
 
-        index_field = self.index_field
-        if index_field != 'index':
-            x_nd_new = self.x_nd[read_slc]
-            x_nd_new[:] = new_from_src[index_field][:, np.newaxis]
-
-        # if (self.x_nd[self.xy_slice][-1] == 0.5).any():
-        #     breakpoint()
-
     def format_xy_nd_to_1d(
         self,
 
@@ -815,15 +831,10 @@ class StepCurveFmtr(IncrementalFormatter):
         x_step = self.x_nd[start:stop]
         y_step = self.y_nd[start:stop]
 
-        # if (x_step[-1] == 0.5).any():
-        #     breakpoint()
-
         # pack in duplicate final value to complete last step level
-        # x_step[-1] = last_t
-        # y_step[-1] = last
-        # x_step[-1, 1] = last_t
         y_step[-1, 1] = last
 
+        # debugging
         # if y_step.any():
         #     s = 3
         #     print(
@@ -833,7 +844,9 @@ class StepCurveFmtr(IncrementalFormatter):
 
         # slice out in-view data
         ivl, ivr = vr
-        # TODO: WHY do we need the extra +1 index?
+
+        # NOTE: add an extra step to get the vertical-line-down-to-zero
+        # adjacent to the last-datum graphic (filled rect).
         x_step_iv = x_step[ivl:ivr+1]
         y_step_iv = y_step[ivl:ivr+1]
 
@@ -847,6 +860,7 @@ class StepCurveFmtr(IncrementalFormatter):
         if x_1d.any() and (x_1d[-1] == 0.5).any():
             breakpoint()
 
+        # debugging
         # if y_1d.any():
         #     s = 6
         #     print(

piker/ui/_render.py

@@ -102,7 +102,6 @@ def render_baritems(
             fmtr=OHLCBarsFmtr(
                 shm=viz.shm,
                 viz=viz,
-                _last_read=read,
                 index_field=viz.index_field,
             ),
         )
@@ -112,7 +111,6 @@ def render_baritems(
             fmtr=OHLCBarsAsCurveFmtr(
                 shm=viz.shm,
                 viz=viz,
-                _last_read=read,
                 index_field=viz.index_field,
             ),
         )
@@ -528,7 +526,6 @@ class Viz(msgspec.Struct):  # , frozen=True):
             # print('exiting early')
             return graphics
 
-        slice_to_head: int = -1
         should_redraw: bool = False
         should_line: bool = False
         rkwargs = {}
@@ -565,7 +562,6 @@ class Viz(msgspec.Struct):  # , frozen=True):
                     fmtr=StepCurveFmtr(
                         shm=self.shm,
                         viz=self,
-                        _last_read=read,
                         index_field=self.index_field,
                     ),
                 )
@@ -573,7 +569,6 @@ class Viz(msgspec.Struct):  # , frozen=True):
                 # TODO: append logic inside ``.render()`` isn't
                 # correct yet for step curves.. remove this to see it.
                 should_redraw = True
-                slice_to_head = -2
 
             else:
                 r = self._src_r
@@ -584,13 +579,9 @@ class Viz(msgspec.Struct):  # , frozen=True):
                         fmtr=IncrementalFormatter(
                             shm=self.shm,
                             viz=self,
-                            _last_read=read,
                         ),
                     )
 
-        if isinstance(graphics, StepCurve):
-            slice_to_head = -2
-
         # ``Curve`` derivative case(s):
         array_key = array_key or self.name
         # print(array_key)
@@ -654,7 +645,6 @@ class Viz(msgspec.Struct):  # , frozen=True):
             should_ds=should_ds,
             showing_src_data=showing_src_data,
 
-            slice_to_head=slice_to_head,
             do_append=do_append,
 
             **rkwargs,
@@ -881,7 +871,6 @@ class Renderer(msgspec.Struct):
         showing_src_data: bool = True,
 
         do_append: bool = True,
-        slice_to_head: int = -1,
         use_fpath: bool = True,
 
         # only render datums "in view" of the ``ChartView``
@@ -921,7 +910,6 @@ class Renderer(msgspec.Struct):
             array_key,
             profiler,
 
-            slice_to_head=slice_to_head,
             read_src_from_key=read_from_key,
             slice_to_inview=use_vr,
         )