From 5a8be0f73d6f60ff08746377eb09ca459f39deab Mon Sep 17 00:00:00 2001 From: Gerd Hoffmann Date: Wed, 13 Jul 2016 12:21:20 +0200 Subject: [PATCH 1/3] vnc: make sure we finish disconnect It may happen that vnc connections linger in disconnecting state forever because VncState happens to be in a state where vnc_update_client() exists early and never reaches the vnc_disconnect_finish() call at the bottom of the function. Fix that by doing an additinal check at the start of the function. https://bugzilla.redhat.com/show_bug.cgi?id=1352799 Signed-off-by: Gerd Hoffmann Message-id: 1468405280-2571-1-git-send-email-kraxel@redhat.com --- ui/vnc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ui/vnc.c b/ui/vnc.c index e3f857cc90..3ce3a5beec 100644 --- a/ui/vnc.c +++ b/ui/vnc.c @@ -1024,6 +1024,11 @@ static int find_and_clear_dirty_height(VncState *vs, static int vnc_update_client(VncState *vs, int has_dirty, bool sync) { + if (vs->disconnecting) { + vnc_disconnect_finish(vs); + return 0; + } + vs->has_dirty += has_dirty; if (vs->need_update && !vs->disconnecting) { VncDisplay *vd = vs->vd; From 3f7e51bca3ef2d64c53b35ab9916c99e4a9e3c69 Mon Sep 17 00:00:00 2001 From: "Herongguang (Stephen)" Date: Tue, 12 Jul 2016 17:31:23 +0800 Subject: [PATCH 2/3] vnc-enc-tight: fix off-by-one bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In tight_encode_indexed_rect32, buf(or src)’s size is count. In for loop, the logic is supposed to be that i is an index into src, i should be incremented when incrementing src. This is broken when src is incremented but i is not before while loop, resulting in off-by-one bug in while loop. Signed-off-by: He Rongguang Message-id: 5784B8EB.7010008@huawei.com Signed-off-by: Gerd Hoffmann --- ui/vnc-enc-tight.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c index b8581dd2e9..877c09319d 100644 --- a/ui/vnc-enc-tight.c +++ b/ui/vnc-enc-tight.c @@ -461,9 +461,10 @@ static int tight_fill_palette(VncState *vs, int x, int y, \ src = (uint##bpp##_t *) buf; \ \ - for (i = 0; i < count; i++) { \ + for (i = 0; i < count; ) { \ \ rgb = *src++; \ + i++; \ rep = 0; \ while (i < count && *src == rgb) { \ rep++, src++, i++; \ From 66668d197fa40747e835e15617eda2f1bc80982f Mon Sep 17 00:00:00 2001 From: Peter Lieven Date: Fri, 15 Jul 2016 11:45:11 +0200 Subject: [PATCH 3/3] vnc-tight: fix regression with libxenstore commit 095497ff added thread local storage for the color counting palette. Unfortunately, a VncPalette is about 7kB on a x86_64 system. This memory is reserved from the stack of every thread and it exhausted the stack space of a libxenstore thread. Fix this by allocating memory only for the VNC encoding thread. Fixes: 095497ffc66b7f031ff2a17f1e50f5cb105ce588 Reported-by: Juergen Gross Tested-by: Juergen Gross Signed-off-by: Peter Lieven Message-id: 1468575911-20656-1-git-send-email-pl@kamp.de Signed-off-by: Gerd Hoffmann --- ui/vnc-enc-tight.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c index 877c09319d..49df85e763 100644 --- a/ui/vnc-enc-tight.c +++ b/ui/vnc-enc-tight.c @@ -1458,11 +1458,17 @@ static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h, } #endif -static __thread VncPalette color_count_palette; +static __thread VncPalette *color_count_palette; +static __thread Notifier vnc_tight_cleanup_notifier; + +static void vnc_tight_cleanup(Notifier *n, void *value) +{ + g_free(color_count_palette); + color_count_palette = NULL; +} static int send_sub_rect(VncState *vs, int x, int y, int w, int h) { - VncPalette *palette = &color_count_palette; uint32_t bg = 0, fg = 0; int colors; int ret = 0; @@ -1471,6 +1477,12 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h) bool allow_jpeg = true; #endif + if (!color_count_palette) { + color_count_palette = g_malloc(sizeof(VncPalette)); + vnc_tight_cleanup_notifier.notify = vnc_tight_cleanup; + qemu_thread_atexit_add(&vnc_tight_cleanup_notifier); + } + vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type); vnc_tight_start(vs); @@ -1491,17 +1503,19 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h) } #endif - colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, palette); + colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, color_count_palette); #ifdef CONFIG_VNC_JPEG if (allow_jpeg && vs->tight.quality != (uint8_t)-1) { - ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, palette, - force_jpeg); + ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, + color_count_palette, force_jpeg); } else { - ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette); + ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, + color_count_palette); } #else - ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette); + ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, + color_count_palette); #endif return ret;