Add CELL_BLITTEDMASK to track ncvisual cells

For properly stacking transparent blittings (#1068), we
need tag those cells which both (1) originated in an
ncvisual operation and (2) have some transparency. For
the three affected blitters (halfblock, quadrant, and
sexblitter), call cell_set_blitted().

include/notcurses/notcurses.h

@@ -109,7 +109,6 @@ API int notcurses_ucs32_to_utf8(const char32_t* ucs32, unsigned ucs32count,
 #define CELL_ALPHA_BLEND        0x10000000ull
 #define CELL_ALPHA_OPAQUE       0x00000000ull
 
-#define CELL_NOBACKGROUND_MASK  0x0400000000000000ull
 // if this bit is set, we are *not* using the default background color
 #define CELL_BGDEFAULT_MASK     0x0000000040000000ull
 // if this bit is set, we are *not* using the default foreground color
@@ -548,6 +547,9 @@ channels_set_bg_default(uint64_t* channels){
 // meaningfully set transparency, but it can be mixed into a cascading color.
 // RGB is used if neither default terminal colors nor palette indexing are in
 // play, and fully supports all transparency options.
+//
+// This structure is exposed only so that most functions can be inlined. Do not
+// directly modify or access the fields of this structure; use the API.
 typedef struct nccell {
   // These 32 bits, together with the associated plane's associated egcpool,
   // completely define this cell's EGC. Unless the EGC requires more than four
@@ -593,14 +595,15 @@ typedef struct nccell {
   // (channels & 0x4000000000000000ull): foreground is *not* "default color"
   // (channels & 0x3000000000000000ull): foreground alpha (2 bits)
   // (channels & 0x0800000000000000ull): foreground uses palette index
-  // (channels & 0x0400000000000000ull): glyph is entirely foreground
+  // (channels & 0x0400000000000000ull): entirely foreground (used for optimization in rasterization)
   // (channels & 0x0300000000000000ull): reserved, must be 0
   // (channels & 0x00ffffff00000000ull): foreground in 3x8 RGB (rrggbb)
   // (channels & 0x0000000080000000ull): reserved, must be 0
   // (channels & 0x0000000040000000ull): background is *not* "default color"
   // (channels & 0x0000000030000000ull): background alpha (2 bits)
   // (channels & 0x0000000008000000ull): background uses palette index
-  // (channels & 0x0000000007000000ull): reserved, must be 0
+  // (channels & 0x0000000004000000ull): drawn by ncvisual (used to trigger blitter stacking)
+  // (channels & 0x0000000003000000ull): reserved, must be 0
   // (channels & 0x0000000000ffffffull): background in 3x8 RGB (rrggbb)
   // At render time, these 24-bit values are quantized down to terminal
   // capabilities, if necessary. There's a clear path to 10-bit support should

src/lib/blit.c

@@ -125,6 +125,7 @@ tria_blit(ncplane* nc, int placey, int placex, int linesize,
       }
       if(ffmpeg_trans_p(rgbbase_up[3]) || ffmpeg_trans_p(rgbbase_down[3])){
         cell_set_bg_alpha(c, CELL_ALPHA_TRANSPARENT);
+        cell_set_blitted(c);
         if(ffmpeg_trans_p(rgbbase_up[3]) && ffmpeg_trans_p(rgbbase_down[3])){
           cell_set_fg_alpha(c, CELL_ALPHA_TRANSPARENT);
         }else if(ffmpeg_trans_p(rgbbase_up[3])){ // down has the color
@@ -433,6 +434,8 @@ quadrant_blit(ncplane* nc, int placey, int placex, int linesize,
           cell_set_bg_alpha(c, CELL_ALPHA_BLEND);
           cell_set_fg_alpha(c, CELL_ALPHA_BLEND);
         }
+      }else{
+        cell_set_blitted(c);
       }
       if(*egc){
         if(pool_blit_direct(&nc->pool, c, egc, strlen(egc), 1) <= 0){
@@ -632,6 +635,8 @@ sextant_blit(ncplane* nc, int placey, int placex, int linesize,
       const char* egc = sex_trans_check(rgbas, &c->channels, blendcolors);
       if(egc == NULL){
         egc = sex_solver(rgbas, &c->channels, blendcolors);
+      }else{
+        cell_set_blitted(c);
       }
 //fprintf(stderr, "sex EGC: %s channels: %016lx\n", egc, c->channels);
       if(*egc){

src/lib/internal.h

@@ -31,6 +31,14 @@ extern "C" {
 struct esctrie;
 struct ncvisual_details;
 
+// Does this glyph completely obscure the background? If so, there's no need
+// to emit a background when rasterizing, a small optimization.
+#define CELL_NOBACKGROUND_MASK  0x0400000000000000ull
+
+// Was this glyph drawn as part of an ncvisual? If so, we need to honor
+// blitter stacking rather than the standard trichannel solver.
+#define CELL_BLITTERSTACK_MASK  0x0000000004000000ull
+
 // we can't define multipart ncvisual here, because OIIO requires C++ syntax,
 // and we can't go throwing C++ syntax into this header. so it goes.
 
@@ -368,31 +376,6 @@ void warn_terminfo(const notcurses* nc, const tinfo* ti);
 
 int resize_callbacks_children(ncplane* n);
 
-// Search the provided multibyte (UTF8) string 's' for the provided unicode
-// codepoint 'cp'. If found, return the column offset of the EGC in which the
-// codepoint appears in 'col', and the byte offset as the return value. If not
-// found, -1 is returned, and 'col' is meaningless.
-static inline int
-mbstr_find_codepoint(const char* s, char32_t cp, int* col){
-  mbstate_t ps;
-  memset(&ps, 0, sizeof(ps));
-  size_t bytes = 0;
-  size_t r;
-  wchar_t w;
-  *col = 0;
-  while((r = mbrtowc(&w, s + bytes, MB_CUR_MAX, &ps)) != (size_t)-1 && r != (size_t)-2){
-    if(r == 0){
-      break;
-    }
-    if(towlower(cp) == towlower(w)){
-      return bytes;
-    }
-    *col += wcwidth(w);
-    bytes += r;
-  }
-  return -1;
-}
-
 static inline ncpile*
 ncplane_pile(ncplane* n){
   return n->pile;
@@ -869,6 +852,21 @@ cell_nobackground_p(const nccell* c){
   return c->channels & CELL_NOBACKGROUND_MASK;
 }
 
+// True if the cell was blitted as part of an ncvisual, and has a transparent
+// background.
+static inline bool
+cell_blitted_p(const nccell* c){
+  return c->channels & CELL_BLITTERSTACK_MASK;
+}
+
+// Set this whenever blitting an ncvisual, when we have a transparent
+// background. In such cases, ncvisuals underneath the cell must be rendered
+// slightly differently.
+static inline void
+cell_set_blitted(nccell* c){
+  c->channels |= CELL_BLITTERSTACK_MASK;
+}
+
 // Destroy a plane and all its bound descendants.
 int ncplane_genocide(ncplane *ncp);
 
@@ -945,9 +943,8 @@ egc_rtl(const char* egc, int* bytes){
 }
 
 // lowest level of cell+pool setup. if the EGC changes the output to RTL, it
-// must be suffixed with a LTR-forcing character by now, and both
-// CELL_WIDEASIAN_MASK and CELL_NOBACKGROUND_MASK ought be set however they're
-// going to be set.
+// must be suffixed with a LTR-forcing character by now, and
+// CELL_NOBACKGROUND_MASK ought be set however it's going to be set.
 static inline int
 pool_blit_direct(egcpool* pool, nccell* c, const char* gcluster, int bytes, int cols){
   pool_release(pool, c);

src/lib/menu.c

@@ -1,5 +1,30 @@
 #include "internal.h"
 
+// Search the provided multibyte (UTF8) string 's' for the provided unicode
+// codepoint 'cp'. If found, return the column offset of the EGC in which the
+// codepoint appears in 'col', and the byte offset as the return value. If not
+// found, -1 is returned, and 'col' is meaningless.
+static int
+mbstr_find_codepoint(const char* s, char32_t cp, int* col){
+  mbstate_t ps;
+  memset(&ps, 0, sizeof(ps));
+  size_t bytes = 0;
+  size_t r;
+  wchar_t w;
+  *col = 0;
+  while((r = mbrtowc(&w, s + bytes, MB_CUR_MAX, &ps)) != (size_t)-1 && r != (size_t)-2){
+    if(r == 0){
+      break;
+    }
+    if(towlower(cp) == towlower(w)){
+      return bytes;
+    }
+    *col += wcwidth(w);
+    bytes += r;
+  }
+  return -1;
+}
+
 static void
 free_menu_section(ncmenu_int_section* ms){
   for(int i = 0 ; i < ms->itemcount ; ++i){