From 6f5009c38cafad883212810a2219479a8ff408c0 Mon Sep 17 00:00:00 2001 From: Zhang Chen Date: Tue, 5 Sep 2017 14:31:04 +0800 Subject: net/colo-compare.c: Optimize unpredictable tcp options comparison When network is busy, some tcp options(like sack) will unpredictable occur in primary side or secondary side. it will make packet size not same, but the two packet's payload is identical. colo just care about packet payload, so we skip the option field. Signed-off-by: Zhang Chen Signed-off-by: Jason Wang --- net/colo-compare.c | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) (limited to 'net/colo-compare.c') diff --git a/net/colo-compare.c b/net/colo-compare.c index 5fe8e3fad9..612fbac72c 100644 --- a/net/colo-compare.c +++ b/net/colo-compare.c @@ -186,7 +186,10 @@ static int packet_enqueue(CompareState *s, int mode) * return: 0 means packet same * > 0 || < 0 means packet different */ -static int colo_packet_compare_common(Packet *ppkt, Packet *spkt, int offset) +static int colo_packet_compare_common(Packet *ppkt, + Packet *spkt, + int poffset, + int soffset) { if (trace_event_get_state_backends(TRACE_COLO_COMPARE_MISCOMPARE)) { char pri_ip_src[20], pri_ip_dst[20], sec_ip_src[20], sec_ip_dst[20]; @@ -201,12 +204,13 @@ static int colo_packet_compare_common(Packet *ppkt, Packet *spkt, int offset) sec_ip_src, sec_ip_dst); } - offset = ppkt->vnet_hdr_len + offset; + poffset = ppkt->vnet_hdr_len + poffset; + soffset = ppkt->vnet_hdr_len + soffset; - if (ppkt->size == spkt->size) { - return memcmp(ppkt->data + offset, - spkt->data + offset, - spkt->size - offset); + if (ppkt->size - poffset == spkt->size - soffset) { + return memcmp(ppkt->data + poffset, + spkt->data + soffset, + spkt->size - soffset); } else { trace_colo_compare_main("Net packet size are not the same"); return -1; @@ -263,13 +267,22 @@ static int colo_packet_compare_tcp(Packet *spkt, Packet *ppkt) * so we just need skip this field. */ if (ptcp->th_off > 5) { - ptrdiff_t tcp_offset; + ptrdiff_t ptcp_offset, stcp_offset; - tcp_offset = ppkt->transport_header - (uint8_t *)ppkt->data - + (ptcp->th_off * 4) - ppkt->vnet_hdr_len; - res = colo_packet_compare_common(ppkt, spkt, tcp_offset); + ptcp_offset = ppkt->transport_header - (uint8_t *)ppkt->data + + (ptcp->th_off * 4) - ppkt->vnet_hdr_len; + stcp_offset = spkt->transport_header - (uint8_t *)spkt->data + + (stcp->th_off * 4) - spkt->vnet_hdr_len; + + /* + * When network is busy, some tcp options(like sack) will unpredictable + * occur in primary side or secondary side. it will make packet size + * not same, but the two packet's payload is identical. colo just + * care about packet payload, so we skip the option field. + */ + res = colo_packet_compare_common(ppkt, spkt, ptcp_offset, stcp_offset); } else if (ptcp->th_sum == stcp->th_sum) { - res = colo_packet_compare_common(ppkt, spkt, ETH_HLEN); + res = colo_packet_compare_common(ppkt, spkt, ETH_HLEN, ETH_HLEN); } else { res = -1; } @@ -330,6 +343,7 @@ static int colo_packet_compare_udp(Packet *spkt, Packet *ppkt) * the ip payload here. */ ret = colo_packet_compare_common(ppkt, spkt, + network_header_length + ETH_HLEN, network_header_length + ETH_HLEN); if (ret) { @@ -367,6 +381,7 @@ static int colo_packet_compare_icmp(Packet *spkt, Packet *ppkt) * the ip payload here. */ if (colo_packet_compare_common(ppkt, spkt, + network_header_length + ETH_HLEN, network_header_length + ETH_HLEN)) { trace_colo_compare_icmp_miscompare("primary pkt size", ppkt->size); @@ -404,7 +419,7 @@ static int colo_packet_compare_other(Packet *spkt, Packet *ppkt) sec_ip_src, sec_ip_dst); } - return colo_packet_compare_common(ppkt, spkt, 0); + return colo_packet_compare_common(ppkt, spkt, 0, 0); } static int colo_old_packet_check_one(Packet *pkt, int64_t *check_time) -- cgit 1.4.1 From 626bba988af8f38b2820f60cfdf5a27bcb74123b Mon Sep 17 00:00:00 2001 From: Zhang Chen Date: Tue, 5 Sep 2017 14:31:05 +0800 Subject: net/colo-compare.c: Adjust net queue pop order for performance The packet_enqueue() use g_queue_push_tail() to enqueue net packet, so it is more efficent way use g_queue_pop_head() to get packet for compare. That will improve the success rate of comparison. In my test the performance of ftp put 1000M file will increase 10% Signed-off-by: Zhang Chen Signed-off-by: Jason Wang --- net/colo-compare.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'net/colo-compare.c') diff --git a/net/colo-compare.c b/net/colo-compare.c index 612fbac72c..f480315753 100644 --- a/net/colo-compare.c +++ b/net/colo-compare.c @@ -484,7 +484,7 @@ static void colo_compare_connection(void *opaque, void *user_data) while (!g_queue_is_empty(&conn->primary_list) && !g_queue_is_empty(&conn->secondary_list)) { - pkt = g_queue_pop_tail(&conn->primary_list); + pkt = g_queue_pop_head(&conn->primary_list); switch (conn->ip_proto) { case IPPROTO_TCP: result = g_queue_find_custom(&conn->secondary_list, @@ -522,7 +522,7 @@ static void colo_compare_connection(void *opaque, void *user_data) * until next comparison. */ trace_colo_compare_main("packet different"); - g_queue_push_tail(&conn->primary_list, pkt); + g_queue_push_head(&conn->primary_list, pkt); /* TODO: colo_notify_checkpoint();*/ break; } -- cgit 1.4.1 From 61c5f46959ee74d6dabd0ee648ca555932f91955 Mon Sep 17 00:00:00 2001 From: Zhang Chen Date: Tue, 5 Sep 2017 14:31:06 +0800 Subject: net/colo-compare.c: Fix comments and scheme Signed-off-by: Zhang Chen Signed-off-by: Jason Wang --- net/colo-compare.c | 59 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 28 deletions(-) (limited to 'net/colo-compare.c') diff --git a/net/colo-compare.c b/net/colo-compare.c index f480315753..eee33b87a8 100644 --- a/net/colo-compare.c +++ b/net/colo-compare.c @@ -41,27 +41,27 @@ #define REGULAR_PACKET_CHECK_MS 3000 /* - + CompareState ++ - | | - +---------------+ +---------------+ +---------------+ - |conn list +--->conn +--------->conn | - +---------------+ +---------------+ +---------------+ - | | | | | | - +---------------+ +---v----+ +---v----+ +---v----+ +---v----+ - |primary | |secondary |primary | |secondary - |packet | |packet + |packet | |packet + - +--------+ +--------+ +--------+ +--------+ - | | | | - +---v----+ +---v----+ +---v----+ +---v----+ - |primary | |secondary |primary | |secondary - |packet | |packet + |packet | |packet + - +--------+ +--------+ +--------+ +--------+ - | | | | - +---v----+ +---v----+ +---v----+ +---v----+ - |primary | |secondary |primary | |secondary - |packet | |packet + |packet | |packet + - +--------+ +--------+ +--------+ +--------+ -*/ + * + CompareState ++ + * | | + * +---------------+ +---------------+ +---------------+ + * | conn list + - > conn + ------- > conn + -- > ...... + * +---------------+ +---------------+ +---------------+ + * | | | | | | + * +---------------+ +---v----+ +---v----+ +---v----+ +---v----+ + * |primary | |secondary |primary | |secondary + * |packet | |packet + |packet | |packet + + * +--------+ +--------+ +--------+ +--------+ + * | | | | + * +---v----+ +---v----+ +---v----+ +---v----+ + * |primary | |secondary |primary | |secondary + * |packet | |packet + |packet | |packet + + * +--------+ +--------+ +--------+ +--------+ + * | | | | + * +---v----+ +---v----+ +---v----+ +---v----+ + * |primary | |secondary |primary | |secondary + * |packet | |packet + |packet | |packet + + * +--------+ +--------+ +--------+ +--------+ + */ typedef struct CompareState { Object parent; @@ -75,14 +75,14 @@ typedef struct CompareState { SocketReadState sec_rs; bool vnet_hdr; - /* connection list: the connections belonged to this NIC could be found - * in this list. - * element type: Connection + /* + * Record the connection that through the NIC + * Element type: Connection */ GQueue conn_list; - /* hashtable to save connection */ + /* Record the connection without repetition */ GHashTable *connection_track_table; - /* compare thread, a thread for each NIC */ + /* This thread just do packet compare job */ QemuThread thread; GMainContext *worker_context; @@ -445,8 +445,11 @@ static int colo_old_packet_check_one_conn(Connection *conn, (GCompareFunc)colo_old_packet_check_one); if (result) { - /* do checkpoint will flush old packet */ - /* TODO: colo_notify_checkpoint();*/ + /* Do checkpoint will flush old packet */ + /* + * TODO: Notify colo frame to do checkpoint. + * colo_compare_inconsistent_notify(); + */ return 0; } -- cgit 1.4.1 From dd321ecfc2e82e6f9578b986060b1aa3f036bd98 Mon Sep 17 00:00:00 2001 From: Wang Yong Date: Tue, 29 Aug 2017 15:22:38 +0800 Subject: colo-compare: Use IOThread to Check old packet regularly and Process pactkets of the primary Remove the task which check old packet in the comparing thread, then use IOthread context timer to handle it. Process pactkets in the IOThread which arrived over the socket. we use iothread_get_g_main_context to create a new g_main_loop in the IOThread.then the packets from the primary and the secondary are processed in the IOThread. Finally remove the colo-compare thread using the IOThread instead. Reviewed-by: Zhang Chen Signed-off-by: Wang Yong Signed-off-by: Wang Guang Signed-off-by: Jason Wang --- net/colo-compare.c | 83 +++++++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 38 deletions(-) (limited to 'net/colo-compare.c') diff --git a/net/colo-compare.c b/net/colo-compare.c index eee33b87a8..b3f35d729a 100644 --- a/net/colo-compare.c +++ b/net/colo-compare.c @@ -29,6 +29,7 @@ #include "qemu/sockets.h" #include "qapi-visit.h" #include "net/colo.h" +#include "sysemu/iothread.h" #define TYPE_COLO_COMPARE "colo-compare" #define COLO_COMPARE(obj) \ @@ -82,11 +83,10 @@ typedef struct CompareState { GQueue conn_list; /* Record the connection without repetition */ GHashTable *connection_track_table; - /* This thread just do packet compare job */ - QemuThread thread; + IOThread *iothread; GMainContext *worker_context; - GMainLoop *compare_loop; + QEMUTimer *packet_check_timer; } CompareState; typedef struct CompareClass { @@ -615,22 +615,40 @@ static void compare_sec_chr_in(void *opaque, const uint8_t *buf, int size) * Check old packet regularly so it can watch for any packets * that the secondary hasn't produced equivalents of. */ -static gboolean check_old_packet_regular(void *opaque) +static void check_old_packet_regular(void *opaque) { CompareState *s = opaque; /* if have old packet we will notify checkpoint */ colo_old_packet_check(s); + timer_mod(s->packet_check_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + + REGULAR_PACKET_CHECK_MS); +} + +static void colo_compare_timer_init(CompareState *s) +{ + AioContext *ctx = iothread_get_aio_context(s->iothread); - return TRUE; + s->packet_check_timer = aio_timer_new(ctx, QEMU_CLOCK_VIRTUAL, + SCALE_MS, check_old_packet_regular, + s); + timer_mod(s->packet_check_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + + REGULAR_PACKET_CHECK_MS); } -static void *colo_compare_thread(void *opaque) +static void colo_compare_timer_del(CompareState *s) { - CompareState *s = opaque; - GSource *timeout_source; + if (s->packet_check_timer) { + timer_del(s->packet_check_timer); + timer_free(s->packet_check_timer); + s->packet_check_timer = NULL; + } + } - s->worker_context = g_main_context_new(); +static void colo_compare_iothread(CompareState *s) +{ + object_ref(OBJECT(s->iothread)); + s->worker_context = iothread_get_g_main_context(s->iothread); qemu_chr_fe_set_handlers(&s->chr_pri_in, compare_chr_can_read, compare_pri_chr_in, NULL, NULL, @@ -639,20 +657,7 @@ static void *colo_compare_thread(void *opaque) compare_sec_chr_in, NULL, NULL, s, s->worker_context, true); - s->compare_loop = g_main_loop_new(s->worker_context, FALSE); - - /* To kick any packets that the secondary doesn't match */ - timeout_source = g_timeout_source_new(REGULAR_PACKET_CHECK_MS); - g_source_set_callback(timeout_source, - (GSourceFunc)check_old_packet_regular, s, NULL); - g_source_attach(timeout_source, s->worker_context); - - g_main_loop_run(s->compare_loop); - - g_source_unref(timeout_source); - g_main_loop_unref(s->compare_loop); - g_main_context_unref(s->worker_context); - return NULL; + colo_compare_timer_init(s); } static char *compare_get_pri_indev(Object *obj, Error **errp) @@ -777,12 +782,10 @@ static void colo_compare_complete(UserCreatable *uc, Error **errp) { CompareState *s = COLO_COMPARE(uc); Chardev *chr; - char thread_name[64]; - static int compare_id; - if (!s->pri_indev || !s->sec_indev || !s->outdev) { + if (!s->pri_indev || !s->sec_indev || !s->outdev || !s->iothread) { error_setg(errp, "colo compare needs 'primary_in' ," - "'secondary_in','outdev' property set"); + "'secondary_in','outdev','iothread' property set"); return; } else if (!strcmp(s->pri_indev, s->outdev) || !strcmp(s->sec_indev, s->outdev) || @@ -817,12 +820,7 @@ static void colo_compare_complete(UserCreatable *uc, Error **errp) g_free, connection_destroy); - sprintf(thread_name, "colo-compare %d", compare_id); - qemu_thread_create(&s->thread, thread_name, - colo_compare_thread, s, - QEMU_THREAD_JOINABLE); - compare_id++; - + colo_compare_iothread(s); return; } @@ -866,6 +864,10 @@ static void colo_compare_init(Object *obj) object_property_add_str(obj, "outdev", compare_get_outdev, compare_set_outdev, NULL); + object_property_add_link(obj, "iothread", TYPE_IOTHREAD, + (Object **)&s->iothread, + object_property_allow_set_link, + OBJ_PROP_LINK_UNREF_ON_RELEASE, NULL); s->vnet_hdr = false; object_property_add_bool(obj, "vnet_hdr_support", compare_get_vnet_hdr, @@ -879,16 +881,21 @@ static void colo_compare_finalize(Object *obj) qemu_chr_fe_deinit(&s->chr_pri_in, false); qemu_chr_fe_deinit(&s->chr_sec_in, false); qemu_chr_fe_deinit(&s->chr_out, false); - - g_main_loop_quit(s->compare_loop); - qemu_thread_join(&s->thread); - + if (s->iothread) { + colo_compare_timer_del(s); + } /* Release all unhandled packets after compare thead exited */ g_queue_foreach(&s->conn_list, colo_flush_packets, s); g_queue_clear(&s->conn_list); - g_hash_table_destroy(s->connection_track_table); + if (s->connection_track_table) { + g_hash_table_destroy(s->connection_track_table); + } + + if (s->iothread) { + object_unref(OBJECT(s->iothread)); + } g_free(s->pri_indev); g_free(s->sec_indev); g_free(s->outdev); -- cgit 1.4.1