From 458fecca80963b4c2c2164889d817542d2cece4f Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Wed, 29 Sep 2021 16:43:10 +0200 Subject: [PATCH 01/11] migration: provide an error message to migration_cancel() This avoids to call migrate_get_current() in the caller function whereas migration_cancel() already needs the pointer to the current migration state. Signed-off-by: Laurent Vivier Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/migration.c | 9 ++++++--- migration/migration.h | 2 +- migration/ram.c | 3 +-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/migration/migration.c b/migration/migration.c index 53b9a8af96..ec3d87f0a9 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -215,8 +215,11 @@ void migration_object_init(void) dirty_bitmap_mig_init(); } -void migration_cancel(void) +void migration_cancel(const Error *error) { + if (error) { + migrate_set_error(current_migration, error); + } migrate_fd_cancel(current_migration); } @@ -226,7 +229,7 @@ void migration_shutdown(void) * Cancel the current migration - that will (eventually) * stop the migration using this structure */ - migration_cancel(); + migration_cancel(NULL); object_unref(OBJECT(current_migration)); /* @@ -2334,7 +2337,7 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk, void qmp_migrate_cancel(Error **errp) { - migration_cancel(); + migration_cancel(NULL); } void qmp_migrate_continue(MigrationStatus state, Error **errp) diff --git a/migration/migration.h b/migration/migration.h index 7a5aa8c2fd..8130b703eb 100644 --- a/migration/migration.h +++ b/migration/migration.h @@ -388,7 +388,7 @@ int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque); void migration_make_urgent_request(void); void migration_consume_urgent_request(void); bool migration_rate_limit(void); -void migration_cancel(void); +void migration_cancel(const Error *error); void populate_vfio_info(MigrationInfo *info); diff --git a/migration/ram.c b/migration/ram.c index 680a5158aa..f5d39db4e4 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -4323,9 +4323,8 @@ static void ram_mig_ram_block_resized(RAMBlockNotifier *n, void *host, * Abort and indicate a proper reason. */ error_setg(&err, "RAM block '%s' resized during precopy.", rb->idstr); - migrate_set_error(migrate_get_current(), err); + migration_cancel(err); error_free(err); - migration_cancel(); } switch (ps) { From fa0b31d585b66e0a9e6e9e72edf729ad2f10e6fd Mon Sep 17 00:00:00 2001 From: yuxiating Date: Thu, 2 Sep 2021 19:51:17 +0800 Subject: [PATCH 02/11] migration: initialise compression_counters for a new migration If the compression migration fails or is canceled, the query for the value of compression_counters during the next compression migration is wrong. Signed-off-by: yuxiating Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/migration.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/migration/migration.c b/migration/migration.c index ec3d87f0a9..edc0dac80a 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -2271,10 +2271,11 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc, migrate_init(s); /* - * set ram_counters memory to zero for a + * set ram_counters compression_counters memory to zero for a * new migration */ memset(&ram_counters, 0, sizeof(ram_counters)); + memset(&compression_counters, 0, sizeof(compression_counters)); return true; } From 02abee3d5129e1430af35ba4a174980a5a22d64e Mon Sep 17 00:00:00 2001 From: Juan Quintela Date: Tue, 2 Nov 2021 16:43:52 +0100 Subject: [PATCH 03/11] migration: Zero migration compression counters Based on previous patch from yuxiating Signed-off-by: Juan Quintela --- migration/savevm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/migration/savevm.c b/migration/savevm.c index 7b7b64bd13..d59e976d50 100644 --- a/migration/savevm.c +++ b/migration/savevm.c @@ -1567,6 +1567,7 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp) migrate_init(ms); memset(&ram_counters, 0, sizeof(ram_counters)); + memset(&compression_counters, 0, sizeof(compression_counters)); ms->to_dst_file = f; qemu_mutex_unlock_iothread(); From ae4c2099351a6ea5774428a0884ace32595b341c Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Mon, 1 Nov 2021 15:56:59 +0800 Subject: [PATCH 04/11] Some minor optimizations for COLO Signed-off-by: Lei Rao Reviewed-by: Dr. David Alan Gilbert Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/colo.c | 2 +- net/colo-compare.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/migration/colo.c b/migration/colo.c index 79fa1f6619..616dc00af7 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -152,7 +152,7 @@ static void primary_vm_do_failover(void) * kick COLO thread which might wait at * qemu_sem_wait(&s->colo_checkpoint_sem). */ - colo_checkpoint_notify(migrate_get_current()); + colo_checkpoint_notify(s); /* * Wake up COLO thread which may blocked in recv() or send(), diff --git a/net/colo-compare.c b/net/colo-compare.c index b100e7b51f..4a64a5d386 100644 --- a/net/colo-compare.c +++ b/net/colo-compare.c @@ -170,7 +170,7 @@ static bool packet_matches_str(const char *str, return false; } - return !memcmp(str, buf, strlen(str)); + return !memcmp(str, buf, packet_len); } static void notify_remote_frame(CompareState *s) From aa505f8e0e04ddb404c00b907b176a6204cbb3c7 Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Mon, 1 Nov 2021 15:57:00 +0800 Subject: [PATCH 05/11] Fixed qemu crash when guest power off in COLO mode This patch fixes the following: qemu-system-x86_64: invalid runstate transition: 'shutdown' -> 'running' Aborted (core dumped) The gdb bt as following: 0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50 1 0x00007faa3d613859 in __GI_abort () at abort.c:79 2 0x000055c5a21268fd in runstate_set (new_state=RUN_STATE_RUNNING) at vl.c:723 3 0x000055c5a1f8cae4 in vm_prepare_start () at /home/workspace/colo-qemu/cpus.c:2206 4 0x000055c5a1f8cb1b in vm_start () at /home/workspace/colo-qemu/cpus.c:2213 5 0x000055c5a2332bba in migration_iteration_finish (s=0x55c5a4658810) at migration/migration.c:3376 6 0x000055c5a2332f3b in migration_thread (opaque=0x55c5a4658810) at migration/migration.c:3527 7 0x000055c5a251d68a in qemu_thread_start (args=0x55c5a5491a70) at util/qemu-thread-posix.c:519 8 0x00007faa3d7e9609 in start_thread (arg=) at pthread_create.c:477 9 0x00007faa3d710293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 Signed-off-by: Lei Rao Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/migration.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/migration/migration.c b/migration/migration.c index edc0dac80a..3fb856f6e1 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -3626,7 +3626,9 @@ static void migration_iteration_finish(MigrationState *s) case MIGRATION_STATUS_CANCELLED: case MIGRATION_STATUS_CANCELLING: if (s->vm_was_running) { - vm_start(); + if (!runstate_check(RUN_STATE_SHUTDOWN)) { + vm_start(); + } } else { if (runstate_check(RUN_STATE_FINISH_MIGRATE)) { runstate_set(RUN_STATE_POSTMIGRATE); From 684bfd1820d45435ee4956e5a3ed231121b36634 Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Mon, 1 Nov 2021 15:57:01 +0800 Subject: [PATCH 06/11] Fixed SVM hang when do failover before PVM crash This patch fixed as follows: Thread 1 (Thread 0x7f34ee738d80 (LWP 11212)): #0 __pthread_clockjoin_ex (threadid=139847152957184, thread_return=0x7f30b1febf30, clockid=, abstime=, block=) at pthread_join_common.c:145 #1 0x0000563401998e36 in qemu_thread_join (thread=0x563402d66610) at util/qemu-thread-posix.c:587 #2 0x00005634017a79fa in process_incoming_migration_co (opaque=0x0) at migration/migration.c:502 #3 0x00005634019b59c9 in coroutine_trampoline (i0=63395504, i1=22068) at util/coroutine-ucontext.c:115 #4 0x00007f34ef860660 in ?? () at ../sysdeps/unix/sysv/linux/x86_64/__start_context.S:91 from /lib/x86_64-linux-gnu/libc.so.6 #5 0x00007f30b21ee730 in ?? () #6 0x0000000000000000 in ?? () Thread 13 (Thread 0x7f30b3dff700 (LWP 11747)): #0 __lll_lock_wait (futex=futex@entry=0x56340218ffa0 , private=0) at lowlevellock.c:52 #1 0x00007f34efa000a3 in _GI__pthread_mutex_lock (mutex=0x56340218ffa0 ) at ../nptl/pthread_mutex_lock.c:80 #2 0x0000563401997f99 in qemu_mutex_lock_impl (mutex=0x56340218ffa0 , file=0x563401b7a80e "migration/colo.c", line=806) at util/qemu-thread-posix.c:78 #3 0x0000563401407144 in qemu_mutex_lock_iothread_impl (file=0x563401b7a80e "migration/colo.c", line=806) at /home/workspace/colo-qemu/cpus.c:1899 #4 0x00005634017ba8e8 in colo_process_incoming_thread (opaque=0x563402d664c0) at migration/colo.c:806 #5 0x0000563401998b72 in qemu_thread_start (args=0x5634039f8370) at util/qemu-thread-posix.c:519 #6 0x00007f34ef9fd609 in start_thread (arg=) at pthread_create.c:477 #7 0x00007f34ef924293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95 The QEMU main thread is holding the lock: (gdb) p qemu_global_mutex $1 = {lock = {_data = {lock = 2, __count = 0, __owner = 11212, __nusers = 9, __kind = 0, __spins = 0, __elision = 0, __list = {_prev = 0x0, __next = 0x0}}, __size = "\002\000\000\000\000\000\000\000\314+\000\000\t", '\000' , __align = 2}, file = 0x563401c07e4b "util/main-loop.c", line = 240, initialized = true} >From the call trace, we can see it is a deadlock bug. and the QEMU main thread holds the global mutex to wait until the COLO thread ends. and the colo thread wants to acquire the global mutex, which will cause a deadlock. So, we should release the qemu_global_mutex before waiting colo thread ends. Signed-off-by: Lei Rao Reviewed-by: Li Zhijian Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/migration.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migration/migration.c b/migration/migration.c index 3fb856f6e1..abaf6f9e3d 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -590,8 +590,10 @@ static void process_incoming_migration_co(void *opaque) mis->have_colo_incoming_thread = true; qemu_coroutine_yield(); + qemu_mutex_unlock_iothread(); /* Wait checkpoint incoming thread exit before free resource */ qemu_thread_join(&mis->colo_incoming_thread); + qemu_mutex_lock_iothread(); /* We hold the global iothread lock, so it is safe here */ colo_release_ram_cache(); } From ac183dac96603005ef2985d0e2ea2eb84fe2e03b Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Mon, 1 Nov 2021 15:57:02 +0800 Subject: [PATCH 07/11] colo: fixed 'Segmentation fault' when the simplex mode PVM poweroff The GDB statck is as follows: Program terminated with signal SIGSEGV, Segmentation fault. 0 object_class_dynamic_cast (class=0x55c8f5d2bf50, typename=0x55c8f2f7379e "qio-channel") at qom/object.c:832 if (type->class->interfaces && [Current thread is 1 (Thread 0x7f756e97eb00 (LWP 1811577))] (gdb) bt 0 object_class_dynamic_cast (class=0x55c8f5d2bf50, typename=0x55c8f2f7379e "qio-channel") at qom/object.c:832 1 0x000055c8f2c3dd14 in object_dynamic_cast (obj=0x55c8f543ac00, typename=0x55c8f2f7379e "qio-channel") at qom/object.c:763 2 0x000055c8f2c3ddce in object_dynamic_cast_assert (obj=0x55c8f543ac00, typename=0x55c8f2f7379e "qio-channel", file=0x55c8f2f73780 "migration/qemu-file-channel.c", line=117, func=0x55c8f2f73800 <__func__.18724> "channel_shutdown") at qom/object.c:786 3 0x000055c8f2bbc6ac in channel_shutdown (opaque=0x55c8f543ac00, rd=true, wr=true, errp=0x0) at migration/qemu-file-channel.c:117 4 0x000055c8f2bba56e in qemu_file_shutdown (f=0x7f7558070f50) at migration/qemu-file.c:67 5 0x000055c8f2ba5373 in migrate_fd_cancel (s=0x55c8f4ccf3f0) at migration/migration.c:1699 6 0x000055c8f2ba1992 in migration_shutdown () at migration/migration.c:187 7 0x000055c8f29a5b77 in main (argc=69, argv=0x7fff3e9e8c08, envp=0x7fff3e9e8e38) at vl.c:4512 The root cause is that we still want to shutdown the from_dst_file in migrate_fd_cancel() after qemu_close in colo_process_checkpoint(). So, we should set the s->rp_state.from_dst_file = NULL after qemu_close(). Signed-off-by: Lei Rao Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/colo.c | 1 + 1 file changed, 1 insertion(+) diff --git a/migration/colo.c b/migration/colo.c index 616dc00af7..907241ab5c 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -640,6 +640,7 @@ out: */ if (s->rp_state.from_dst_file) { qemu_fclose(s->rp_state.from_dst_file); + s->rp_state.from_dst_file = NULL; } } From 04dd89169b94789aacde0a9b29943e8614879343 Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Mon, 1 Nov 2021 15:57:03 +0800 Subject: [PATCH 08/11] Removed the qemu_fclose() in colo_process_incoming_thread After the live migration, the related fd will be cleanup in migration_incoming_state_destroy(). So, the qemu_close() in colo_process_incoming_thread is not necessary. Signed-off-by: Lei Rao Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/colo.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/migration/colo.c b/migration/colo.c index 907241ab5c..71fc82a040 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -919,11 +919,6 @@ out: /* Hope this not to be too long to loop here */ qemu_sem_wait(&mis->colo_incoming_sem); qemu_sem_destroy(&mis->colo_incoming_sem); - /* Must be called after failover BH is completed */ - if (mis->to_src_file) { - qemu_fclose(mis->to_src_file); - mis->to_src_file = NULL; - } rcu_unregister_thread(); return NULL; From 2b9f6bf36c7483a07e45cc20cb0bb794769fb6d1 Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Mon, 1 Nov 2021 15:57:04 +0800 Subject: [PATCH 09/11] Changed the last-mode to none of first start COLO When we first stated the COLO, the last-mode is as follows: { "execute": "query-colo-status" } {"return": {"last-mode": "primary", "mode": "primary", "reason": "none"}} The last-mode is unreasonable. After the patch, will be changed to the following: { "execute": "query-colo-status" } {"return": {"last-mode": "none", "mode": "primary", "reason": "none"}} Signed-off-by: Lei Rao Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/colo.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/migration/colo.c b/migration/colo.c index 71fc82a040..e3b1f136f4 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -205,7 +205,7 @@ void colo_do_failover(void) vm_stop_force_state(RUN_STATE_COLO); } - switch (get_colo_mode()) { + switch (last_colo_mode = get_colo_mode()) { case COLO_MODE_PRIMARY: primary_vm_do_failover(); break; @@ -530,8 +530,7 @@ static void colo_process_checkpoint(MigrationState *s) Error *local_err = NULL; int ret; - last_colo_mode = get_colo_mode(); - if (last_colo_mode != COLO_MODE_PRIMARY) { + if (get_colo_mode() != COLO_MODE_PRIMARY) { error_report("COLO mode must be COLO_MODE_PRIMARY"); return; } @@ -830,8 +829,7 @@ void *colo_process_incoming_thread(void *opaque) migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, MIGRATION_STATUS_COLO); - last_colo_mode = get_colo_mode(); - if (last_colo_mode != COLO_MODE_SECONDARY) { + if (get_colo_mode() != COLO_MODE_SECONDARY) { error_report("COLO mode must be COLO_MODE_SECONDARY"); return NULL; } From e5fdf920964b65678798960d8b3a55453c2e9094 Mon Sep 17 00:00:00 2001 From: Lukas Straub Date: Sun, 4 Jul 2021 18:14:44 +0200 Subject: [PATCH 10/11] colo: Don't dump colo cache if dump-guest-core=off One might set dump-guest-core=off to make coredumps smaller and still allow to debug many qemu bugs. Extend this option to the colo cache. Signed-off-by: Lukas Straub Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- migration/ram.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/migration/ram.c b/migration/ram.c index f5d39db4e4..847af461f2 100644 --- a/migration/ram.c +++ b/migration/ram.c @@ -56,6 +56,8 @@ #include "multifd.h" #include "sysemu/runstate.h" +#include "hw/boards.h" /* for machine_dump_guest_core() */ + #if defined(__linux__) #include "qemu/userfaultfd.h" #endif /* defined(__linux__) */ @@ -3542,6 +3544,10 @@ int colo_init_ram_cache(void) } return -errno; } + if (!machine_dump_guest_core(current_machine)) { + qemu_madvise(block->colo_cache, block->used_length, + QEMU_MADV_DONTDUMP); + } } } From 64153ca613d0a50d1301eae4bd895aade001fcca Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Wed, 3 Nov 2021 10:21:12 +0800 Subject: [PATCH 11/11] Optimized the function of fill_connection_key. Remove some unnecessary code to improve the performance of the filter-rewriter module. Signed-off-by: Lei Rao Reviewed-by: Zhang Chen Reviewed-by: Juan Quintela Signed-off-by: Juan Quintela --- net/colo-compare.c | 2 +- net/colo.c | 31 ++++++++++++------------------- net/colo.h | 6 +++--- net/filter-rewriter.c | 10 +--------- 4 files changed, 17 insertions(+), 32 deletions(-) diff --git a/net/colo-compare.c b/net/colo-compare.c index 4a64a5d386..b8876d7fd9 100644 --- a/net/colo-compare.c +++ b/net/colo-compare.c @@ -264,7 +264,7 @@ static int packet_enqueue(CompareState *s, int mode, Connection **con) pkt = NULL; return -1; } - fill_connection_key(pkt, &key); + fill_connection_key(pkt, &key, false); conn = connection_get(s->connection_track_table, &key, diff --git a/net/colo.c b/net/colo.c index 3a3e6e89a0..1f8162f59f 100644 --- a/net/colo.c +++ b/net/colo.c @@ -83,19 +83,26 @@ int parse_packet_early(Packet *pkt) return 0; } -void extract_ip_and_port(uint32_t tmp_ports, ConnectionKey *key, Packet *pkt) +void extract_ip_and_port(uint32_t tmp_ports, ConnectionKey *key, + Packet *pkt, bool reverse) { + if (reverse) { + key->src = pkt->ip->ip_dst; + key->dst = pkt->ip->ip_src; + key->src_port = ntohs(tmp_ports & 0xffff); + key->dst_port = ntohs(tmp_ports >> 16); + } else { key->src = pkt->ip->ip_src; key->dst = pkt->ip->ip_dst; key->src_port = ntohs(tmp_ports >> 16); key->dst_port = ntohs(tmp_ports & 0xffff); + } } -void fill_connection_key(Packet *pkt, ConnectionKey *key) +void fill_connection_key(Packet *pkt, ConnectionKey *key, bool reverse) { - uint32_t tmp_ports; + uint32_t tmp_ports = 0; - memset(key, 0, sizeof(*key)); key->ip_proto = pkt->ip->ip_p; switch (key->ip_proto) { @@ -106,29 +113,15 @@ void fill_connection_key(Packet *pkt, ConnectionKey *key) case IPPROTO_SCTP: case IPPROTO_UDPLITE: tmp_ports = *(uint32_t *)(pkt->transport_header); - extract_ip_and_port(tmp_ports, key, pkt); break; case IPPROTO_AH: tmp_ports = *(uint32_t *)(pkt->transport_header + 4); - extract_ip_and_port(tmp_ports, key, pkt); break; default: break; } -} -void reverse_connection_key(ConnectionKey *key) -{ - struct in_addr tmp_ip; - uint16_t tmp_port; - - tmp_ip = key->src; - key->src = key->dst; - key->dst = tmp_ip; - - tmp_port = key->src_port; - key->src_port = key->dst_port; - key->dst_port = tmp_port; + extract_ip_and_port(tmp_ports, key, pkt, reverse); } Connection *connection_new(ConnectionKey *key) diff --git a/net/colo.h b/net/colo.h index d91cd245c4..8b3e8d5a83 100644 --- a/net/colo.h +++ b/net/colo.h @@ -89,9 +89,9 @@ typedef struct Connection { uint32_t connection_key_hash(const void *opaque); int connection_key_equal(const void *opaque1, const void *opaque2); int parse_packet_early(Packet *pkt); -void extract_ip_and_port(uint32_t tmp_ports, ConnectionKey *key, Packet *pkt); -void fill_connection_key(Packet *pkt, ConnectionKey *key); -void reverse_connection_key(ConnectionKey *key); +void extract_ip_and_port(uint32_t tmp_ports, ConnectionKey *key, + Packet *pkt, bool reverse); +void fill_connection_key(Packet *pkt, ConnectionKey *key, bool reverse); Connection *connection_new(ConnectionKey *key); void connection_destroy(void *opaque); Connection *connection_get(GHashTable *connection_track_table, diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c index cb3a96cde1..bf05023dc3 100644 --- a/net/filter-rewriter.c +++ b/net/filter-rewriter.c @@ -279,15 +279,7 @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf, */ if (pkt && is_tcp_packet(pkt)) { - fill_connection_key(pkt, &key); - - if (sender == nf->netdev) { - /* - * We need make tcp TX and RX packet - * into one connection. - */ - reverse_connection_key(&key); - } + fill_connection_key(pkt, &key, sender == nf->netdev); /* After failover we needn't change new TCP packet */ if (s->failover_mode &&