This is the real fragment of SpotRemover's code
that does intensity/color locking.

So, we have 3 pixel pairs:

mm0 - pixel A
mm1 - pixel B
mm5 - pixel C

Locking takes place if

|A-C|<MaxStable  and
|B-C|<MaxStable

then  B = C (copying only bytes with variation smaller than MaxStable)


Each pixel in the 64-bit pixel pair has 32 bits:

U - 8-bit (higher order) chroma 1  (unsigned)
V - 8-bit                chroma 2  (unsigned)
0 - 8-bit empty but also processed
Y - 8-bit (lower order)  luma, unsigned


LockInten      dw  0304h          ; (LockInten=4, LockColor=3)
dummy          dw  ????h          ; (just any variable)
......

ToSignedByte   dq  8080808080808080h
MaxStable      dq  0000000000000000h
MaxStableNeg   dq  0000000000000000h


;  INIT part    (can be slow)

; Preparing the MaxStable vars

    mov       eax,  dword ptr LockInten
    movd      mm0,  eax         ;     0 0 0 0     ?  ?  LColor LIinten
    punpcklbw mm0,  mm0         ;     ? ? ? ?     LC LC   LI  LI
    punpckldq mm0,  mm0         ;    LC LC LI LI  LC LC   LI  LI
    movq      MaxStable, mm0    ; save MaxStable
    pcmpeqw   mm1,  mm1
    pxor      mm0,  mm1         ;   -MaxStable-1 (MaxStableNeg)
    movq      MaxStableNeg, mm0


................

   ; MAIN LOOP  (must be fast)


   ; we need to convert pixels to signed bytes because unsigned (A-C) can be over 7F, and
   ; then pcmpgtb will treat is as a negative byte.
   ; if we do psubusb D, (A-C) then we need also to compare to 0.

      psubb     mm0,  ToSignedByte ; 8-bit signed   A
      psubb     mm5,  ToSignedByte ; 8-bit signed   B
      psubb     mm1,  ToSignedByte ; 8-bit signed   C

   ;  Calculate the differences
   ;
      psubsb    mm0,  mm1          ;  A-C  intensity difference between the last and the oldest
      movq      mm2,  mm0          ;  A-C
   ;
   ;  Compare to produce the mask
   ;
      pcmpgtb   mm0,  MaxStable    ;  A-C>M     if TRUE then unstable
      pcmpgtb   mm2,  MaxStableNeg ;  A-C>-M-1  if TRUE then stable (means not(A-C<-M))
      pandn     mm0,  mm2          ;  FF => stable


   ;  Calculate the differences
   ;
      movq      mm3,  mm5
      psubsb    mm3,  mm1          ;  B-C  intensity difference between the older and the oldest
      movq      mm2,  mm3          ;  B-C
   ;
   ;  Compare to produce the mask
   ;
      pcmpgtb   mm3,  MaxStable    ;  B-C>M     if TRUE then unstable
      pcmpgtb   mm2,  MaxStableNeg ;  B-C>-M-1  if TRUE then stable (means not(B-C<-M))
      pandn     mm3,  mm2          ;  FF => stable

      pand      mm0,  mm3          ; final mask    FF - stable; 00 - unstable
   ;
   ;  Copy the bytes with smaller difference
   ;
      pxor      mm1,  mm5          ;  C xor B
      pand      mm1,  mm0          ;  C xor B (<=MaxStable) | 0 (>MaxStable)
      pxor      mm1,  mm5          ;  C xor B xor B | B
      psubb     mm1, ToSignedByte  ;  8-bit unsigned

      movq     Locked_PixelPair, mm1    ;  save somewhere