try to handle OOM on "termpty_save"

src/bin/termpty.c

@@ -613,35 +613,47 @@ termpty_line_length(const Termcell *cells, ssize_t nb_cells)
 }
 
 static int
-termpty_line_find_top(Termpty *ty, int y_end)
+termpty_line_find_top(Termpty *ty, int y_end, int *top)
 {
    int y_start = y_end;
-   Termsave *ts;
 
    while (y_start > 0)
      {
         if (TERMPTY_SCREEN(ty, ty->w - 1, y_start - 1).att.autowrapped)
           y_start--;
         else
-          return y_start;
+          {
+             *top = y_start;
+             return 0;
+          }
      }
    while (-y_start < ty->backscroll_num)
      {
-        ts = termpty_save_extract(ty->back[(y_start + ty->backpos - 1 +
-                                            ty->backmax) % ty->backmax]);
+        Termsave *ts = ty->back[(y_start + ty->backpos - 1 +
+                                 ty->backmax) % ty->backmax];
+        if (ts)
+          {
+             ts = termpty_save_extract(ts);
+             if (!ts)
+               return -1;
+          }
         ty->back[(y_start + ty->backpos - 1 + ty->backmax) % ty->backmax] = ts;
         if (ts->cell[ts->w - 1].att.autowrapped)
           y_start--;
         else
-          return y_start;
+          {
+             *top = y_start;
+             return 0;
+          }
      }
-   return y_start;
+   *top = y_start;
+   return 0;
 }
 
 static int
 termpty_line_rewrap(Termpty *ty, int y_start, int y_end,
                     Termcell *screen2, Termsave **back2,
-                    int w2, int y2_end)
+                    int w2, int y2_end, int *new_y2_start)
 {
    int x, x2, y, y2, y2_start;
    int len, len_last, len_remaining, copy_width, ts2_width;
@@ -656,6 +668,8 @@ termpty_line_rewrap(Termpty *ty, int y_start, int y_end,
      {
         ts = termpty_save_extract(ty->back[(y_end + ty->backpos +
                                             ty->backmax) % ty->backmax]);
+        if (!ts)
+          return -1;
         ty->back[(y_end + ty->backpos + ty->backmax) % ty->backmax] = ts;
         len_last = ts->w;
      }
@@ -669,7 +683,8 @@ termpty_line_rewrap(Termpty *ty, int y_start, int y_end,
      {
         if (y2_start < 0)
           back2[y2_start + ty->backmax] = termpty_save_new(0);
-        return y2_start;
+        *new_y2_start = y2_start;
+        return 0;
      }
    if (-y2_start > ty->backmax)
      {
@@ -696,6 +711,8 @@ termpty_line_rewrap(Termpty *ty, int y_start, int y_end,
           {
              ts = termpty_save_extract(ty->back[(y + ty->backpos +
                                                  ty->backmax) % ty->backmax]);
+             if (!ts)
+               return -1;
              ty->back[(y + ty->backpos + ty->backmax) % ty->backmax] = ts;
              line = ts->cell;
           }
@@ -738,15 +755,16 @@ termpty_line_rewrap(Termpty *ty, int y_start, int y_end,
         x = 0;
         y++;
      }
-   return y2_start;
+   *new_y2_start = y2_start;
+   return 0;
 }
 
 
 void
 termpty_resize(Termpty *ty, int new_w, int new_h)
 {
-   Termcell *new_screen;
-   Termsave **new_back;
+   Termcell *new_screen = NULL;
+   Termsave **new_back = NULL;
    int y_start, y_end, new_y_start = 0, new_y_end;
    int i, altbuf = 0;
 
@@ -764,35 +782,33 @@ termpty_resize(Termpty *ty, int new_w, int new_h)
 
    new_screen = calloc(1, sizeof(Termcell) * new_w * new_h);
    if (!new_screen)
-     {
-        ERR("memerr");
-        return;
-     }
+     goto bad;
    free(ty->screen2);
    ty->screen2 = calloc(1, sizeof(Termcell) * new_w * new_h);
    if (!ty->screen2)
-     {
-        ERR("memerr");
-        free(new_screen);
-        return;
-     }
+     goto bad;
    new_back = calloc(sizeof(Termsave *), ty->backmax);
 
    y_end = ty->state.cy;
    new_y_end = new_h - 1;
    while ((y_end >= -ty->backscroll_num) && (new_y_end >= -ty->backmax))
      {
-        y_start = termpty_line_find_top(ty, y_end);
-        new_y_start = termpty_line_rewrap(ty, y_start, y_end, new_screen,
-                                        new_back, new_w, new_y_end);
+        if (termpty_line_find_top(ty, y_end, &y_start) < 0)
+          goto bad;
+        if (termpty_line_rewrap(ty, y_start, y_end, new_screen,
+                                new_back, new_w, new_y_end,
+                                &new_y_start) < 0)
+          goto bad;
         y_end = y_start - 1;
         new_y_end = new_y_start - 1;
      }
 
    free(ty->screen);
+   ty->screen = new_screen;
    for (i = 1; i <= ty->backscroll_num; i++)
      termpty_save_free(ty->back[(ty->backpos - i + ty->backmax) % ty->backmax]);
    free(ty->back);
+   ty->back = new_back;
 
    ty->w = new_w;
    ty->h = new_h;
@@ -803,8 +819,6 @@ termpty_resize(Termpty *ty, int new_w, int new_h)
    ty->backpos = 0;
    ty->backscroll_num = MAX(-new_y_start, 0);
    ty->state.had_cr = 0;
-   ty->screen = new_screen;
-   ty->back = new_back;
 
    if (altbuf) termpty_screen_swap(ty);
 
@@ -815,6 +829,13 @@ termpty_resize(Termpty *ty, int new_w, int new_h)
    _pty_size(ty);
 
    termpty_save_thaw();
+   return;
+
+bad:
+   termpty_save_thaw();
+   free(new_screen);
+   free(new_back);
+
 }
 
 void

src/bin/termptyops.c

@@ -42,6 +42,8 @@ termpty_text_save_top(Termpty *ty, Termcell *cells, ssize_t w_max)
    termpty_save_freeze();
    w = termpty_line_length(cells, w_max);
    ts = termpty_save_new(w);
+   if (!ts)
+     return;
    termpty_cell_copy(ty, cells, ts->cell, w);
    if (!ty->back) ty->back = calloc(1, sizeof(Termsave *) * ty->backmax);
    if (ty->back[ty->backpos])