From 94fd9cbaa319049d43107fc2128fae803f8a6989 Mon Sep 17 00:00:00 2001 From: Laurent Vivier Date: Fri, 9 Jun 2017 13:08:10 +0200 Subject: spapr: Treat devices added before inbound migration as coldplugged When migrating a guest which has already had devices hotplugged, libvirt typically starts the destination qemu with -incoming defer, adds those hotplugged devices with qmp, then initiates the incoming migration. This causes problems for the management of spapr DRC state. Because the device is treated as hotplugged, it goes into a DRC state for a device immediately after it's plugged, but before the guest has acknowledged its presence. However, chances are the guest on the source machine *has* acknowledged the device's presence and configured it. If the source has fully configured the device, then DRC state won't be sent in the migration stream: for maximum migration compatibility with earlier versions we don't migrate DRCs in coldplug-equivalent state. That means that the DRC effectively changes state over the migrate, causing problems later on. In addition, logging hotplug events for these devices isn't what we want because a) those events should already have been issued on the source host and b) the event queue should get wiped out by the incoming state anyway. In short, what we really want is to treat devices added before an incoming migration as if they were coldplugged. To do this, we first add a spapr_drc_hotplugged() helper which determines if the device is hotplugged in the sense relevant for DRC state management. We only send hotplug events when this is true. Second, when we add a device which isn't hotplugged in this sense, we force a reset of the DRC state - this ensures the DRC is in a coldplug-equivalent state (there isn't usually a system reset between these device adds and the incoming migration). This is based on an earlier patch by Laurent Vivier, cleaned up and extended. Signed-off-by: Laurent Vivier Signed-off-by: David Gibson Reviewed-by: Greg Kurz Tested-by: Daniel Barboza --- hw/ppc/spapr_pci.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'hw/ppc/spapr_pci.c') diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c index a52dcf8ec0..1e84c552d8 100644 --- a/hw/ppc/spapr_pci.c +++ b/hw/ppc/spapr_pci.c @@ -1443,7 +1443,9 @@ static void spapr_pci_plug(HotplugHandler *plug_handler, /* If this is function 0, signal hotplug for all the device functions. * Otherwise defer sending the hotplug event. */ - if (plugged_dev->hotplugged && PCI_FUNC(pdev->devfn) == 0) { + if (!spapr_drc_hotplugged(plugged_dev)) { + spapr_drc_reset(drc); + } else if (PCI_FUNC(pdev->devfn) == 0) { int i; for (i = 0; i < 8; i++) { -- cgit 1.4.1 From a8dc47fd8249cca45f6314bfd8cf195dc3fbd36b Mon Sep 17 00:00:00 2001 From: David Gibson Date: Tue, 4 Jul 2017 21:07:14 +1000 Subject: spapr: Refactor spapr_drc_detach() This function has two unused parameters - remove them. It also sets awaiting_release on all paths, except one. On that path setting it is harmless, since it will be immediately cleared by spapr_drc_release(). So factor it out of the if statements. Signed-off-by: David Gibson Reviewed-by: Greg Kurz Tested-by: Daniel Barboza --- hw/ppc/spapr.c | 11 +++-------- hw/ppc/spapr_drc.c | 12 ++++++------ hw/ppc/spapr_pci.c | 7 +------ include/hw/ppc/spapr_drc.h | 2 +- 4 files changed, 11 insertions(+), 21 deletions(-) (limited to 'hw/ppc/spapr_pci.c') diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c index 8818647e1a..16d450f21c 100644 --- a/hw/ppc/spapr.c +++ b/hw/ppc/spapr.c @@ -2654,7 +2654,7 @@ static void spapr_add_lmbs(DeviceState *dev, uint64_t addr_start, uint64_t size, addr -= SPAPR_MEMORY_BLOCK_SIZE; drc = spapr_drc_by_id(TYPE_SPAPR_DRC_LMB, addr / SPAPR_MEMORY_BLOCK_SIZE); - spapr_drc_detach(drc, dev, NULL); + spapr_drc_detach(drc); } g_free(fdt); error_propagate(errp, local_err); @@ -2876,7 +2876,7 @@ static void spapr_memory_unplug_request(HotplugHandler *hotplug_dev, addr / SPAPR_MEMORY_BLOCK_SIZE); g_assert(drc); - spapr_drc_detach(drc, dev, errp); + spapr_drc_detach(drc); addr += SPAPR_MEMORY_BLOCK_SIZE; } @@ -2942,7 +2942,6 @@ void spapr_core_unplug_request(HotplugHandler *hotplug_dev, DeviceState *dev, { int index; sPAPRDRConnector *drc; - Error *local_err = NULL; CPUCore *cc = CPU_CORE(dev); int smt = kvmppc_smt_threads(); @@ -2959,11 +2958,7 @@ void spapr_core_unplug_request(HotplugHandler *hotplug_dev, DeviceState *dev, drc = spapr_drc_by_id(TYPE_SPAPR_DRC_CPU, index * smt); g_assert(drc); - spapr_drc_detach(drc, dev, &local_err); - if (local_err) { - error_propagate(errp, local_err); - return; - } + spapr_drc_detach(drc); spapr_hotplug_req_remove_by_index(drc); } diff --git a/hw/ppc/spapr_drc.c b/hw/ppc/spapr_drc.c index fd5614094d..15a3d00049 100644 --- a/hw/ppc/spapr_drc.c +++ b/hw/ppc/spapr_drc.c @@ -70,7 +70,7 @@ static uint32_t drc_isolate_physical(sPAPRDRConnector *drc) uint32_t drc_index = spapr_drc_index(drc); if (drc->configured) { trace_spapr_drc_set_isolation_state_finalizing(drc_index); - spapr_drc_detach(drc, DEVICE(drc->dev), NULL); + spapr_drc_detach(drc); } else { trace_spapr_drc_set_isolation_state_deferring(drc_index); } @@ -134,7 +134,7 @@ static uint32_t drc_isolate_logical(sPAPRDRConnector *drc) uint32_t drc_index = spapr_drc_index(drc); if (drc->configured) { trace_spapr_drc_set_isolation_state_finalizing(drc_index); - spapr_drc_detach(drc, DEVICE(drc->dev), NULL); + spapr_drc_detach(drc); } else { trace_spapr_drc_set_isolation_state_deferring(drc_index); } @@ -187,7 +187,7 @@ static uint32_t drc_set_unusable(sPAPRDRConnector *drc) if (drc->awaiting_release) { uint32_t drc_index = spapr_drc_index(drc); trace_spapr_drc_set_allocation_state_finalizing(drc_index); - spapr_drc_detach(drc, DEVICE(drc->dev), NULL); + spapr_drc_detach(drc); } return RTAS_OUT_SUCCESS; @@ -371,20 +371,20 @@ static void spapr_drc_release(sPAPRDRConnector *drc) drc->dev = NULL; } -void spapr_drc_detach(sPAPRDRConnector *drc, DeviceState *d, Error **errp) +void spapr_drc_detach(sPAPRDRConnector *drc) { trace_spapr_drc_detach(spapr_drc_index(drc)); + drc->awaiting_release = true; + if (drc->isolation_state != SPAPR_DR_ISOLATION_STATE_ISOLATED) { trace_spapr_drc_awaiting_isolated(spapr_drc_index(drc)); - drc->awaiting_release = true; return; } if (spapr_drc_type(drc) != SPAPR_DR_CONNECTOR_TYPE_PCI && drc->allocation_state != SPAPR_DR_ALLOCATION_STATE_UNUSABLE) { trace_spapr_drc_awaiting_unusable(spapr_drc_index(drc)); - drc->awaiting_release = true; return; } diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c index 1e84c552d8..092a2f5f3d 100644 --- a/hw/ppc/spapr_pci.c +++ b/hw/ppc/spapr_pci.c @@ -1478,7 +1478,6 @@ static void spapr_pci_unplug_request(HotplugHandler *plug_handler, PCIDevice *pdev = PCI_DEVICE(plugged_dev); sPAPRDRConnectorClass *drck; sPAPRDRConnector *drc = spapr_phb_get_pci_drc(phb, pdev); - Error *local_err = NULL; if (!phb->dr_enabled) { error_setg(errp, QERR_BUS_NO_HOTPLUG, @@ -1516,11 +1515,7 @@ static void spapr_pci_unplug_request(HotplugHandler *plug_handler, } } - spapr_drc_detach(drc, DEVICE(pdev), &local_err); - if (local_err) { - error_propagate(errp, local_err); - return; - } + spapr_drc_detach(drc); /* if this isn't func 0, defer unplug event. otherwise signal removal * for all present functions diff --git a/include/hw/ppc/spapr_drc.h b/include/hw/ppc/spapr_drc.h index 18a196e831..fc8b721639 100644 --- a/include/hw/ppc/spapr_drc.h +++ b/include/hw/ppc/spapr_drc.h @@ -242,6 +242,6 @@ int spapr_drc_populate_dt(void *fdt, int fdt_offset, Object *owner, void spapr_drc_attach(sPAPRDRConnector *drc, DeviceState *d, void *fdt, int fdt_start_offset, Error **errp); -void spapr_drc_detach(sPAPRDRConnector *drc, DeviceState *d, Error **errp); +void spapr_drc_detach(sPAPRDRConnector *drc); #endif /* HW_SPAPR_DRC_H */ -- cgit 1.4.1 From f1c52354e5bdab6983d13a4c174759c585e834b3 Mon Sep 17 00:00:00 2001 From: David Gibson Date: Tue, 20 Jun 2017 21:02:41 +0800 Subject: spapr: Cleanups relating to DRC awaiting_release field 'awaiting_release' indicates that the host has requested an unplug of the device attached to the DRC, but the guest has not (yet) put the device into a state where it is safe to complete removal. 1. Rename it to 'unplug_requested' which to me at least is clearer 2. Remove the ->release_pending() method used to check this from outside spapr_drc.c. The method only plausibly has one implementation, so use a plain function (spapr_drc_unplug_requested()) instead. 3. Remove it from the migration stream. Attempting to migrate mid-unplug is broken not just for spapr - in general management has no good way to determine if the device should be present on the destination or not. So, until that's fixed, there's no point adding extra things to the stream. Signed-off-by: David Gibson Reviewed-by: Greg Kurz Tested-by: Daniel Barboza --- hw/ppc/spapr_drc.c | 26 +++++++++----------------- hw/ppc/spapr_pci.c | 6 ++---- include/hw/ppc/spapr_drc.h | 11 ++++++----- 3 files changed, 17 insertions(+), 26 deletions(-) (limited to 'hw/ppc/spapr_pci.c') diff --git a/hw/ppc/spapr_drc.c b/hw/ppc/spapr_drc.c index 15a3d00049..03ef75c70c 100644 --- a/hw/ppc/spapr_drc.c +++ b/hw/ppc/spapr_drc.c @@ -66,7 +66,7 @@ static uint32_t drc_isolate_physical(sPAPRDRConnector *drc) * configured state, as suggested by the state diagram from PAPR+ * 2.7, 13.4 */ - if (drc->awaiting_release) { + if (drc->unplug_requested) { uint32_t drc_index = spapr_drc_index(drc); if (drc->configured) { trace_spapr_drc_set_isolation_state_finalizing(drc_index); @@ -116,7 +116,7 @@ static uint32_t drc_isolate_logical(sPAPRDRConnector *drc) * actually being unplugged, fail the isolation request here. */ if (spapr_drc_type(drc) == SPAPR_DR_CONNECTOR_TYPE_LMB - && !drc->awaiting_release) { + && !drc->unplug_requested) { return RTAS_OUT_HW_ERROR; } @@ -130,7 +130,7 @@ static uint32_t drc_isolate_logical(sPAPRDRConnector *drc) * configured state, as suggested by the state diagram from PAPR+ * 2.7, 13.4 */ - if (drc->awaiting_release) { + if (drc->unplug_requested) { uint32_t drc_index = spapr_drc_index(drc); if (drc->configured) { trace_spapr_drc_set_isolation_state_finalizing(drc_index); @@ -170,7 +170,7 @@ static uint32_t drc_set_usable(sPAPRDRConnector *drc) if (!drc->dev) { return RTAS_OUT_NO_SUCH_INDICATOR; } - if (drc->awaiting_release) { + if (drc->unplug_requested) { /* Don't allow the guest to move a device away from UNUSABLE * state when we want to unplug it */ return RTAS_OUT_NO_SUCH_INDICATOR; @@ -184,7 +184,7 @@ static uint32_t drc_set_usable(sPAPRDRConnector *drc) static uint32_t drc_set_unusable(sPAPRDRConnector *drc) { drc->allocation_state = SPAPR_DR_ALLOCATION_STATE_UNUSABLE; - if (drc->awaiting_release) { + if (drc->unplug_requested) { uint32_t drc_index = spapr_drc_index(drc); trace_spapr_drc_set_allocation_state_finalizing(drc_index); spapr_drc_detach(drc); @@ -363,7 +363,7 @@ static void spapr_drc_release(sPAPRDRConnector *drc) drck->release(drc->dev); - drc->awaiting_release = false; + drc->unplug_requested = false; g_free(drc->fdt); drc->fdt = NULL; drc->fdt_start_offset = 0; @@ -375,7 +375,7 @@ void spapr_drc_detach(sPAPRDRConnector *drc) { trace_spapr_drc_detach(spapr_drc_index(drc)); - drc->awaiting_release = true; + drc->unplug_requested = true; if (drc->isolation_state != SPAPR_DR_ISOLATION_STATE_ISOLATED) { trace_spapr_drc_awaiting_isolated(spapr_drc_index(drc)); @@ -391,11 +391,6 @@ void spapr_drc_detach(sPAPRDRConnector *drc) spapr_drc_release(drc); } -static bool release_pending(sPAPRDRConnector *drc) -{ - return drc->awaiting_release; -} - void spapr_drc_reset(sPAPRDRConnector *drc) { trace_spapr_drc_reset(spapr_drc_index(drc)); @@ -406,7 +401,7 @@ void spapr_drc_reset(sPAPRDRConnector *drc) /* immediately upon reset we can safely assume DRCs whose devices * are pending removal can be safely removed. */ - if (drc->awaiting_release) { + if (drc->unplug_requested) { spapr_drc_release(drc); } @@ -454,7 +449,7 @@ static bool spapr_drc_needed(void *opaque) case SPAPR_DR_CONNECTOR_TYPE_LMB: rc = !((drc->isolation_state == SPAPR_DR_ISOLATION_STATE_UNISOLATED) && (drc->allocation_state == SPAPR_DR_ALLOCATION_STATE_USABLE) && - drc->configured && !drc->awaiting_release); + drc->configured); break; case SPAPR_DR_CONNECTOR_TYPE_PHB: case SPAPR_DR_CONNECTOR_TYPE_VIO: @@ -474,7 +469,6 @@ static const VMStateDescription vmstate_spapr_drc = { VMSTATE_UINT32(allocation_state, sPAPRDRConnector), VMSTATE_UINT32(dr_indicator, sPAPRDRConnector), VMSTATE_BOOL(configured, sPAPRDRConnector), - VMSTATE_BOOL(awaiting_release, sPAPRDRConnector), VMSTATE_END_OF_LIST() } }; @@ -565,11 +559,9 @@ static void spapr_dr_connector_instance_init(Object *obj) static void spapr_dr_connector_class_init(ObjectClass *k, void *data) { DeviceClass *dk = DEVICE_CLASS(k); - sPAPRDRConnectorClass *drck = SPAPR_DR_CONNECTOR_CLASS(k); dk->realize = realize; dk->unrealize = unrealize; - drck->release_pending = release_pending; /* * Reason: it crashes FIXME find and document the real reason */ diff --git a/hw/ppc/spapr_pci.c b/hw/ppc/spapr_pci.c index 092a2f5f3d..6ecdf29d28 100644 --- a/hw/ppc/spapr_pci.c +++ b/hw/ppc/spapr_pci.c @@ -1476,7 +1476,6 @@ static void spapr_pci_unplug_request(HotplugHandler *plug_handler, { sPAPRPHBState *phb = SPAPR_PCI_HOST_BRIDGE(DEVICE(plug_handler)); PCIDevice *pdev = PCI_DEVICE(plugged_dev); - sPAPRDRConnectorClass *drck; sPAPRDRConnector *drc = spapr_phb_get_pci_drc(phb, pdev); if (!phb->dr_enabled) { @@ -1488,8 +1487,7 @@ static void spapr_pci_unplug_request(HotplugHandler *plug_handler, g_assert(drc); g_assert(drc->dev == plugged_dev); - drck = SPAPR_DR_CONNECTOR_GET_CLASS(drc); - if (!drck->release_pending(drc)) { + if (!spapr_drc_unplug_requested(drc)) { PCIBus *bus = PCI_BUS(qdev_get_parent_bus(DEVICE(pdev))); uint32_t slotnr = PCI_SLOT(pdev->devfn); sPAPRDRConnector *func_drc; @@ -1505,7 +1503,7 @@ static void spapr_pci_unplug_request(HotplugHandler *plug_handler, func_drck = SPAPR_DR_CONNECTOR_GET_CLASS(func_drc); state = func_drck->dr_entity_sense(func_drc); if (state == SPAPR_DR_ENTITY_SENSE_PRESENT - && !func_drck->release_pending(func_drc)) { + && !spapr_drc_unplug_requested(func_drc)) { error_setg(errp, "PCI: slot %d, function %d still present. " "Must unplug all non-0 functions first.", diff --git a/include/hw/ppc/spapr_drc.h b/include/hw/ppc/spapr_drc.h index fc8b721639..5fa502e465 100644 --- a/include/hw/ppc/spapr_drc.h +++ b/include/hw/ppc/spapr_drc.h @@ -199,10 +199,9 @@ typedef struct sPAPRDRConnector { bool configured; sPAPRConfigureConnectorState *ccs; - bool awaiting_release; - /* device pointer, via link property */ DeviceState *dev; + bool unplug_requested; } sPAPRDRConnector; typedef struct sPAPRDRConnectorClass { @@ -218,9 +217,6 @@ typedef struct sPAPRDRConnectorClass { uint32_t (*isolate)(sPAPRDRConnector *drc); uint32_t (*unisolate)(sPAPRDRConnector *drc); void (*release)(DeviceState *dev); - - /* QEMU interfaces for managing hotplug operations */ - bool (*release_pending)(sPAPRDRConnector *drc); } sPAPRDRConnectorClass; static inline bool spapr_drc_hotplugged(DeviceState *dev) @@ -244,4 +240,9 @@ void spapr_drc_attach(sPAPRDRConnector *drc, DeviceState *d, void *fdt, int fdt_start_offset, Error **errp); void spapr_drc_detach(sPAPRDRConnector *drc); +static inline bool spapr_drc_unplug_requested(sPAPRDRConnector *drc) +{ + return drc->unplug_requested; +} + #endif /* HW_SPAPR_DRC_H */ -- cgit 1.4.1