From 765cca0533ca3031f0cd1222eff187b330b527e6 Mon Sep 17 00:00:00 2001
From: Christian Borntraeger <borntraeger@de.ibm.com>
Date: Tue, 10 Jan 2017 14:44:31 +0100
Subject: [PATCH 01/12] s390x: remove double compat statement

We chain our compat handler via the CCW_COMPAT macros and via the
class_init function. (e.g. ccw_machine_2_7_class_options calls
ccw_machine_2_8_class_options). As all class_init functions in that
chain call SET_MACHINE_COMPAT for their compat settings, and
SET_MACHINE_COMPAT will append there is no need to do that again.

Signed-off-by: Christian Borntraeger <borntraeger@de.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/s390-virtio-ccw.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index e340eab36b..dbf4f01165 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -339,7 +339,6 @@ static const TypeInfo ccw_machine_info = {
         HW_COMPAT_2_7
 
 #define CCW_COMPAT_2_6 \
-        CCW_COMPAT_2_7 \
         HW_COMPAT_2_6 \
         {\
             .driver   = TYPE_S390_IPL,\
@@ -352,7 +351,6 @@ static const TypeInfo ccw_machine_info = {
         },
 
 #define CCW_COMPAT_2_5 \
-        CCW_COMPAT_2_6 \
         HW_COMPAT_2_5
 
 #define CCW_COMPAT_2_4 \

From 113725a6a37f01bd4b659cbb5831c8d71d9c673e Mon Sep 17 00:00:00 2001
From: Cornelia Huck <cornelia.huck@de.ibm.com>
Date: Wed, 30 Nov 2016 15:52:46 +0100
Subject: [PATCH 02/12] s390x: add compat machine for 2.9

Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
Acked-by: Christian Borntraeger <borntraeger@de.ibm.com>
---
 hw/s390x/s390-virtio-ccw.c | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index dbf4f01165..e9a676797a 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -335,6 +335,9 @@ static const TypeInfo ccw_machine_info = {
     }                                                                         \
     type_init(ccw_machine_register_##suffix)
 
+#define CCW_COMPAT_2_8 \
+        HW_COMPAT_2_8
+
 #define CCW_COMPAT_2_7 \
         HW_COMPAT_2_7
 
@@ -393,14 +396,26 @@ static const TypeInfo ccw_machine_info = {
             .value    = "0",\
         },
 
+static void ccw_machine_2_9_instance_options(MachineState *machine)
+{
+}
+
+static void ccw_machine_2_9_class_options(MachineClass *mc)
+{
+}
+DEFINE_CCW_MACHINE(2_9, "2.9", true);
+
 static void ccw_machine_2_8_instance_options(MachineState *machine)
 {
+    ccw_machine_2_9_instance_options(machine);
 }
 
 static void ccw_machine_2_8_class_options(MachineClass *mc)
 {
+    ccw_machine_2_9_class_options(mc);
+    SET_MACHINE_COMPAT(mc, CCW_COMPAT_2_8);
 }
-DEFINE_CCW_MACHINE(2_8, "2.8", true);
+DEFINE_CCW_MACHINE(2_8, "2.8", false);
 
 static void ccw_machine_2_7_instance_options(MachineState *machine)
 {

From 4cbd6c41fa3aa901e12e8158e8d22dd8f70f7a90 Mon Sep 17 00:00:00 2001
From: Fei Li <sherrylf@linux.vnet.ibm.com>
Date: Fri, 25 Nov 2016 07:59:07 +0100
Subject: [PATCH 03/12] s390x/kvm: use kvm_gsi_routing_enabled in flic

Let's use kvm_gsi_routing_enabled() to check if kvm supports
KVM_CAP_IRQ_ROUTING in order to avoid a needless ioctl invocation.

Signed-off-by: Fei Li <sherrylf@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/intc/s390_flic_kvm.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hw/intc/s390_flic_kvm.c b/hw/intc/s390_flic_kvm.c
index 21ac2e2dcd..c313166fbe 100644
--- a/hw/intc/s390_flic_kvm.c
+++ b/hw/intc/s390_flic_kvm.c
@@ -201,7 +201,7 @@ static int kvm_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
         .addr = (uint64_t)&adapter,
     };
 
-    if (!kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING)) {
+    if (!kvm_gsi_routing_enabled()) {
         /* nothing to do */
         return 0;
     }
@@ -226,7 +226,7 @@ static int kvm_s390_io_adapter_map(S390FLICState *fs, uint32_t id,
     KVMS390FLICState *flic = KVM_S390_FLIC(fs);
     int r;
 
-    if (!kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING)) {
+    if (!kvm_gsi_routing_enabled()) {
         /* nothing to do */
         return 0;
     }

From de91ea92e9082609dc413f0109abc77283b4ff0a Mon Sep 17 00:00:00 2001
From: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Date: Thu, 8 Dec 2016 13:02:24 +0800
Subject: [PATCH 04/12] s390x/pci: make S390PCIIOMMU inherit Object

Currently S390PCIIOMMU is a normal struct. Let's make it inherit Object
in order to take advantage of QOM. In addition, we move some stuff
related to IOMMU from S390PCIBusDevice to S390PCIIOMMU.

Signed-off-by: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Acked-by: Pierre Morel <pmorel@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/s390-pci-bus.c  | 52 ++++++++++++++++++++++++----------------
 hw/s390x/s390-pci-bus.h  | 20 ++++++++++------
 hw/s390x/s390-pci-inst.c | 50 +++++++++++++++++++-------------------
 hw/s390x/s390-pci-inst.h |  2 +-
 4 files changed, 71 insertions(+), 53 deletions(-)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 63f6248f1d..11244fc93c 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -187,8 +187,8 @@ void s390_pci_sclp_deconfigure(SCCB *sccb)
         if (pbdev->summary_ind) {
             pci_dereg_irqs(pbdev);
         }
-        if (pbdev->iommu_enabled) {
-            pci_dereg_ioat(pbdev);
+        if (pbdev->iommu->enabled) {
+            pci_dereg_ioat(pbdev->iommu);
         }
         pbdev->state = ZPCI_FS_STANDBY;
         rc = SCLP_RC_NORMAL_COMPLETION;
@@ -377,12 +377,12 @@ out:
     return pte;
 }
 
-static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
+static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *mr, hwaddr addr,
                                           bool is_write)
 {
     uint64_t pte;
     uint32_t flags;
-    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, iommu_mr);
+    S390PCIIOMMU *iommu = container_of(mr, S390PCIIOMMU, iommu_mr);
     IOMMUTLBEntry ret = {
         .target_as = &address_space_memory,
         .iova = 0,
@@ -391,10 +391,10 @@ static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
         .perm = IOMMU_NONE,
     };
 
-    switch (pbdev->state) {
+    switch (iommu->pbdev->state) {
     case ZPCI_FS_ENABLED:
     case ZPCI_FS_BLOCKED:
-        if (!pbdev->iommu_enabled) {
+        if (!iommu->enabled) {
             return ret;
         }
         break;
@@ -404,11 +404,11 @@ static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
 
     DPRINTF("iommu trans addr 0x%" PRIx64 "\n", addr);
 
-    if (addr < pbdev->pba || addr > pbdev->pal) {
+    if (addr < iommu->pba || addr > iommu->pal) {
         return ret;
     }
 
-    pte = s390_guest_io_table_walk(s390_pci_get_table_origin(pbdev->g_iota),
+    pte = s390_guest_io_table_walk(s390_pci_get_table_origin(iommu->g_iota),
                                    addr);
     if (!pte) {
         return ret;
@@ -503,19 +503,21 @@ static const MemoryRegionOps s390_msi_ctrl_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-void s390_pci_iommu_enable(S390PCIBusDevice *pbdev)
+void s390_pci_iommu_enable(S390PCIIOMMU *iommu)
 {
-    memory_region_init_iommu(&pbdev->iommu_mr, OBJECT(&pbdev->iommu->mr),
-                             &s390_iommu_ops, "iommu-s390", pbdev->pal + 1);
-    memory_region_add_subregion(&pbdev->iommu->mr, 0, &pbdev->iommu_mr);
-    pbdev->iommu_enabled = true;
+    char *name = g_strdup_printf("iommu-s390-%04x", iommu->pbdev->uid);
+    memory_region_init_iommu(&iommu->iommu_mr, OBJECT(&iommu->mr),
+                             &s390_iommu_ops, name, iommu->pal + 1);
+    iommu->enabled = true;
+    memory_region_add_subregion(&iommu->mr, 0, &iommu->iommu_mr);
+    g_free(name);
 }
 
-void s390_pci_iommu_disable(S390PCIBusDevice *pbdev)
+void s390_pci_iommu_disable(S390PCIIOMMU *iommu)
 {
-    memory_region_del_subregion(&pbdev->iommu->mr, &pbdev->iommu_mr);
-    object_unparent(OBJECT(&pbdev->iommu_mr));
-    pbdev->iommu_enabled = false;
+    iommu->enabled = false;
+    memory_region_del_subregion(&iommu->mr, &iommu->iommu_mr);
+    object_unparent(OBJECT(&iommu->iommu_mr));
 }
 
 static void s390_pcihost_init_as(S390pciState *s)
@@ -660,6 +662,7 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
 
         pbdev->pdev = pdev;
         pbdev->iommu = s->iommu[PCI_SLOT(pdev->devfn)];
+        pbdev->iommu->pbdev = pbdev;
         pbdev->state = ZPCI_FS_STANDBY;
 
         s390_pci_msix_init(pbdev);
@@ -692,8 +695,8 @@ static void s390_pcihost_timer_cb(void *opaque)
     if (pbdev->summary_ind) {
         pci_dereg_irqs(pbdev);
     }
-    if (pbdev->iommu_enabled) {
-        pci_dereg_ioat(pbdev);
+    if (pbdev->iommu->enabled) {
+        pci_dereg_ioat(pbdev->iommu);
     }
 
     pbdev->state = ZPCI_FS_STANDBY;
@@ -877,8 +880,8 @@ static void s390_pci_device_reset(DeviceState *dev)
     if (pbdev->summary_ind) {
         pci_dereg_irqs(pbdev);
     }
-    if (pbdev->iommu_enabled) {
-        pci_dereg_ioat(pbdev);
+    if (pbdev->iommu->enabled) {
+        pci_dereg_ioat(pbdev->iommu);
     }
 
     pbdev->fmb_addr = 0;
@@ -944,11 +947,18 @@ static const TypeInfo s390_pci_device_info = {
     .class_init = s390_pci_device_class_init,
 };
 
+static TypeInfo s390_pci_iommu_info = {
+    .name = TYPE_S390_PCI_IOMMU,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(S390PCIIOMMU),
+};
+
 static void s390_pci_register_types(void)
 {
     type_register_static(&s390_pcihost_info);
     type_register_static(&s390_pcibus_info);
     type_register_static(&s390_pci_device_info);
+    type_register_static(&s390_pci_iommu_info);
 }
 
 type_init(s390_pci_register_types)
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index 7f2701301e..1d05c1a92e 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -23,6 +23,7 @@
 #define TYPE_S390_PCI_HOST_BRIDGE "s390-pcihost"
 #define TYPE_S390_PCI_BUS "s390-pcibus"
 #define TYPE_S390_PCI_DEVICE "zpci"
+#define TYPE_S390_PCI_IOMMU "s390-pci-iommu"
 #define FH_MASK_ENABLE   0x80000000
 #define FH_MASK_INSTANCE 0x7f000000
 #define FH_MASK_SHM      0x00ff0000
@@ -42,6 +43,8 @@
     OBJECT_CHECK(S390PCIBus, (obj), TYPE_S390_PCI_BUS)
 #define S390_PCI_DEVICE(obj) \
     OBJECT_CHECK(S390PCIBusDevice, (obj), TYPE_S390_PCI_DEVICE)
+#define S390_PCI_IOMMU(obj) \
+    OBJECT_CHECK(S390PCIIOMMU, (obj), TYPE_S390_PCI_IOMMU)
 
 #define HP_EVENT_TO_CONFIGURED        0x0301
 #define HP_EVENT_RESERVED_TO_STANDBY  0x0302
@@ -258,24 +261,28 @@ typedef struct S390MsixInfo {
     uint32_t pba_offset;
 } S390MsixInfo;
 
+typedef struct S390PCIBusDevice S390PCIBusDevice;
 typedef struct S390PCIIOMMU {
+    Object parent_obj;
+    S390PCIBusDevice *pbdev;
     AddressSpace as;
     MemoryRegion mr;
+    MemoryRegion iommu_mr;
+    bool enabled;
+    uint64_t g_iota;
+    uint64_t pba;
+    uint64_t pal;
 } S390PCIIOMMU;
 
 typedef struct S390PCIBusDevice {
     DeviceState qdev;
     PCIDevice *pdev;
     ZpciState state;
-    bool iommu_enabled;
     char *target;
     uint16_t uid;
     uint32_t fh;
     uint32_t fid;
     bool fid_defined;
-    uint64_t g_iota;
-    uint64_t pba;
-    uint64_t pal;
     uint64_t fmb_addr;
     uint8_t isc;
     uint16_t noi;
@@ -283,7 +290,6 @@ typedef struct S390PCIBusDevice {
     S390MsixInfo msix;
     AdapterRoutes routes;
     S390PCIIOMMU *iommu;
-    MemoryRegion iommu_mr;
     MemoryRegion msix_notify_mr;
     IndAddr *summary_ind;
     IndAddr *indicator;
@@ -306,8 +312,8 @@ int chsc_sei_nt2_get_event(void *res);
 int chsc_sei_nt2_have_event(void);
 void s390_pci_sclp_configure(SCCB *sccb);
 void s390_pci_sclp_deconfigure(SCCB *sccb);
-void s390_pci_iommu_enable(S390PCIBusDevice *pbdev);
-void s390_pci_iommu_disable(S390PCIBusDevice *pbdev);
+void s390_pci_iommu_enable(S390PCIIOMMU *iommu);
+void s390_pci_iommu_disable(S390PCIIOMMU *iommu);
 void s390_pci_generate_error_event(uint16_t pec, uint32_t fh, uint32_t fid,
                                    uint64_t faddr, uint32_t e);
 S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx);
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 0864d9be12..7dbdb0c6d6 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -555,6 +555,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
     CPUS390XState *env = &cpu->env;
     uint32_t fh;
     S390PCIBusDevice *pbdev;
+    S390PCIIOMMU *iommu;
     hwaddr start, end;
     IOMMUTLBEntry entry;
     MemoryRegion *mr;
@@ -597,7 +598,8 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
         break;
     }
 
-    if (!pbdev->g_iota) {
+    iommu = pbdev->iommu;
+    if (!iommu->g_iota) {
         pbdev->state = ZPCI_FS_ERROR;
         setcc(cpu, ZPCI_PCI_LS_ERR);
         s390_set_status_code(env, r1, ZPCI_PCI_ST_INSUF_RES);
@@ -606,7 +608,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
         goto out;
     }
 
-    if (end < pbdev->pba || start > pbdev->pal) {
+    if (end < iommu->pba || start > iommu->pal) {
         pbdev->state = ZPCI_FS_ERROR;
         setcc(cpu, ZPCI_PCI_LS_ERR);
         s390_set_status_code(env, r1, ZPCI_PCI_ST_INSUF_RES);
@@ -615,7 +617,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
         goto out;
     }
 
-    mr = &pbdev->iommu_mr;
+    mr = &iommu->iommu_mr;
     while (start < end) {
         entry = mr->iommu_ops->translate(mr, start, 0);
 
@@ -783,7 +785,7 @@ int pci_dereg_irqs(S390PCIBusDevice *pbdev)
     return 0;
 }
 
-static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib)
+static int reg_ioat(CPUS390XState *env, S390PCIIOMMU *iommu, ZpciFib fib)
 {
     uint64_t pba = ldq_p(&fib.pba);
     uint64_t pal = ldq_p(&fib.pal);
@@ -803,21 +805,21 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib)
         return -EINVAL;
     }
 
-    pbdev->pba = pba;
-    pbdev->pal = pal;
-    pbdev->g_iota = g_iota;
+    iommu->pba = pba;
+    iommu->pal = pal;
+    iommu->g_iota = g_iota;
 
-    s390_pci_iommu_enable(pbdev);
+    s390_pci_iommu_enable(iommu);
 
     return 0;
 }
 
-void pci_dereg_ioat(S390PCIBusDevice *pbdev)
+void pci_dereg_ioat(S390PCIIOMMU *iommu)
 {
-    s390_pci_iommu_disable(pbdev);
-    pbdev->pba = 0;
-    pbdev->pal = 0;
-    pbdev->g_iota = 0;
+    s390_pci_iommu_disable(iommu);
+    iommu->pba = 0;
+    iommu->pal = 0;
+    iommu->g_iota = 0;
 }
 
 int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
@@ -892,10 +894,10 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
         if (dmaas != 0) {
             cc = ZPCI_PCI_LS_ERR;
             s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL);
-        } else if (pbdev->iommu_enabled) {
+        } else if (pbdev->iommu->enabled) {
             cc = ZPCI_PCI_LS_ERR;
             s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
-        } else if (reg_ioat(env, pbdev, fib)) {
+        } else if (reg_ioat(env, pbdev->iommu, fib)) {
             cc = ZPCI_PCI_LS_ERR;
             s390_set_status_code(env, r1, ZPCI_MOD_ST_INSUF_RES);
         }
@@ -904,23 +906,23 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
         if (dmaas != 0) {
             cc = ZPCI_PCI_LS_ERR;
             s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL);
-        } else if (!pbdev->iommu_enabled) {
+        } else if (!pbdev->iommu->enabled) {
             cc = ZPCI_PCI_LS_ERR;
             s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
         } else {
-            pci_dereg_ioat(pbdev);
+            pci_dereg_ioat(pbdev->iommu);
         }
         break;
     case ZPCI_MOD_FC_REREG_IOAT:
         if (dmaas != 0) {
             cc = ZPCI_PCI_LS_ERR;
             s390_set_status_code(env, r1, ZPCI_MOD_ST_DMAAS_INVAL);
-        } else if (!pbdev->iommu_enabled) {
+        } else if (!pbdev->iommu->enabled) {
             cc = ZPCI_PCI_LS_ERR;
             s390_set_status_code(env, r1, ZPCI_MOD_ST_SEQUENCE);
         } else {
-            pci_dereg_ioat(pbdev);
-            if (reg_ioat(env, pbdev, fib)) {
+            pci_dereg_ioat(pbdev->iommu);
+            if (reg_ioat(env, pbdev->iommu, fib)) {
                 cc = ZPCI_PCI_LS_ERR;
                 s390_set_status_code(env, r1, ZPCI_MOD_ST_INSUF_RES);
             }
@@ -1015,7 +1017,7 @@ int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
         fib.fc |= 0x40;
     case ZPCI_FS_ENABLED:
         fib.fc |= 0x80;
-        if (pbdev->iommu_enabled) {
+        if (pbdev->iommu->enabled) {
             fib.fc |= 0x10;
         }
         if (!(fh & FH_MASK_ENABLE)) {
@@ -1028,9 +1030,9 @@ int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
         return 0;
     }
 
-    stq_p(&fib.pba, pbdev->pba);
-    stq_p(&fib.pal, pbdev->pal);
-    stq_p(&fib.iota, pbdev->g_iota);
+    stq_p(&fib.pba, pbdev->iommu->pba);
+    stq_p(&fib.pal, pbdev->iommu->pal);
+    stq_p(&fib.iota, pbdev->iommu->g_iota);
     stq_p(&fib.aibv, pbdev->routes.adapter.ind_addr);
     stq_p(&fib.aisb, pbdev->routes.adapter.summary_addr);
     stq_p(&fib.fmb_addr, pbdev->fmb_addr);
diff --git a/hw/s390x/s390-pci-inst.h b/hw/s390x/s390-pci-inst.h
index 23f4bfa0ed..94a959f91c 100644
--- a/hw/s390x/s390-pci-inst.h
+++ b/hw/s390x/s390-pci-inst.h
@@ -292,7 +292,7 @@ typedef struct ZpciFib {
 } QEMU_PACKED ZpciFib;
 
 int pci_dereg_irqs(S390PCIBusDevice *pbdev);
-void pci_dereg_ioat(S390PCIBusDevice *pbdev);
+void pci_dereg_ioat(S390PCIIOMMU *iommu);
 int clp_service_call(S390CPU *cpu, uint8_t r2);
 int pcilg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2);
 int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2);

From 03805be08195fca358b12dacc26370f8d1b2c27b Mon Sep 17 00:00:00 2001
From: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Date: Thu, 8 Dec 2016 13:03:18 +0800
Subject: [PATCH 05/12] s390x/pci: dynamically allocate iommu

When initializing a PCI device, an address space is required during PCI
core initialization and before the call to the embedding object hotplug
callback. To provide this AS, we allocate a S390PCIIOMMU object
containing this AS. Initialization of S390PCIIOMMU object is done
before the PCI device is completely created. So that we cannot
associate the IOMMU with the device at the moment. To track the IOMMU
object, we use g_hash functions with the PCI device's bus address as a
key to provide an array of pointers indexed by the PCI device's devfn
to the allocated IOMMU.

Signed-off-by: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Reviewed-by: Pierre Morel <pmorel@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/s390-pci-bus.c | 72 +++++++++++++++++++++++++++++++++--------
 hw/s390x/s390-pci-bus.h |  7 +++-
 2 files changed, 64 insertions(+), 15 deletions(-)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 11244fc93c..f3afb17f6a 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -432,11 +432,48 @@ static const MemoryRegionIOMMUOps s390_iommu_ops = {
     .translate = s390_translate_iommu,
 };
 
+static S390PCIIOMMU *s390_pci_get_iommu(S390pciState *s, PCIBus *bus,
+                                        int devfn)
+{
+    uint64_t key = (uintptr_t)bus;
+    S390PCIIOMMUTable *table = g_hash_table_lookup(s->iommu_table, &key);
+    S390PCIIOMMU *iommu;
+
+    if (!table) {
+        table = g_malloc0(sizeof(S390PCIIOMMUTable));
+        table->key = key;
+        g_hash_table_insert(s->iommu_table, &table->key, table);
+    }
+
+    iommu = table->iommu[PCI_SLOT(devfn)];
+    if (!iommu) {
+        iommu = S390_PCI_IOMMU(object_new(TYPE_S390_PCI_IOMMU));
+
+        char *mr_name = g_strdup_printf("iommu-root-%02x:%02x.%01x",
+                                        pci_bus_num(bus),
+                                        PCI_SLOT(devfn),
+                                        PCI_FUNC(devfn));
+        char *as_name = g_strdup_printf("iommu-pci-%02x:%02x.%01x",
+                                        pci_bus_num(bus),
+                                        PCI_SLOT(devfn),
+                                        PCI_FUNC(devfn));
+        memory_region_init(&iommu->mr, OBJECT(iommu), mr_name, UINT64_MAX);
+        address_space_init(&iommu->as, &iommu->mr, as_name);
+        table->iommu[PCI_SLOT(devfn)] = iommu;
+
+        g_free(mr_name);
+        g_free(as_name);
+    }
+
+    return iommu;
+}
+
 static AddressSpace *s390_pci_dma_iommu(PCIBus *bus, void *opaque, int devfn)
 {
     S390pciState *s = opaque;
+    S390PCIIOMMU *iommu = s390_pci_get_iommu(s, bus, devfn);
 
-    return &s->iommu[PCI_SLOT(devfn)]->as;
+    return &iommu->as;
 }
 
 static uint8_t set_ind_atomic(uint64_t ind_loc, uint8_t to_be_set)
@@ -520,19 +557,21 @@ void s390_pci_iommu_disable(S390PCIIOMMU *iommu)
     object_unparent(OBJECT(&iommu->iommu_mr));
 }
 
-static void s390_pcihost_init_as(S390pciState *s)
+static void s390_pci_iommu_free(PCIBus *bus, int32_t devfn)
 {
-    int i;
-    S390PCIIOMMU *iommu;
+    uint64_t key = (uintptr_t)bus;
+    S390PCIIOMMUTable *table = g_hash_table_lookup(s->iommu_table, &key);
+    S390PCIIOMMU *iommu = table ? table->iommu[PCI_SLOT(devfn)] : NULL;
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        iommu = g_malloc0(sizeof(S390PCIIOMMU));
-        memory_region_init(&iommu->mr, OBJECT(s),
-                           "iommu-root-s390", UINT64_MAX);
-        address_space_init(&iommu->as, &iommu->mr, "iommu-pci");
-
-        s->iommu[i] = iommu;
+    if (!table || !iommu) {
+        return;
     }
+
+    table->iommu[PCI_SLOT(devfn)] = NULL;
+    address_space_destroy(&iommu->as);
+    object_unparent(OBJECT(&iommu->mr));
+    object_unparent(OBJECT(iommu));
+    object_unref(OBJECT(iommu));
 }
 
 static int s390_pcihost_init(SysBusDevice *dev)
@@ -548,7 +587,6 @@ static int s390_pcihost_init(SysBusDevice *dev)
                          s390_pci_set_irq, s390_pci_map_irq, NULL,
                          get_system_memory(), get_system_io(), 0, 64,
                          TYPE_PCI_BUS);
-    s390_pcihost_init_as(s);
     pci_setup_iommu(b, s390_pci_dma_iommu, s);
 
     bus = BUS(b);
@@ -558,6 +596,8 @@ static int s390_pcihost_init(SysBusDevice *dev)
     s->bus = S390_PCI_BUS(qbus_create(TYPE_S390_PCI_BUS, DEVICE(s), NULL));
     qbus_set_hotplug_handler(BUS(s->bus), DEVICE(s), NULL);
 
+    s->iommu_table = g_hash_table_new_full(g_int64_hash, g_int64_equal,
+                                           NULL, g_free);
     QTAILQ_INIT(&s->pending_sei);
     return 0;
 }
@@ -661,7 +701,7 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
         }
 
         pbdev->pdev = pdev;
-        pbdev->iommu = s->iommu[PCI_SLOT(pdev->devfn)];
+        pbdev->iommu = s390_pci_get_iommu(s, pdev->bus, pdev->devfn);
         pbdev->iommu->pbdev = pbdev;
         pbdev->state = ZPCI_FS_STANDBY;
 
@@ -708,8 +748,9 @@ static void s390_pcihost_timer_cb(void *opaque)
 static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
                                     DeviceState *dev, Error **errp)
 {
-    int i;
     PCIDevice *pci_dev = NULL;
+    PCIBus *bus;
+    int32_t devfn, i;
     S390PCIBusDevice *pbdev = NULL;
     S390pciState *s = s390_get_phb();
 
@@ -752,8 +793,11 @@ static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
 
     s390_pci_generate_plug_event(HP_EVENT_STANDBY_TO_RESERVED,
                                  pbdev->fh, pbdev->fid);
+    bus = pci_dev->bus;
+    devfn = pci_dev->devfn;
     object_unparent(OBJECT(pci_dev));
     s390_pci_msix_free(pbdev);
+    s390_pci_iommu_free(bus, devfn);
     pbdev->pdev = NULL;
     pbdev->state = ZPCI_FS_RESERVED;
 out:
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index 1d05c1a92e..fe108e9330 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -274,6 +274,11 @@ typedef struct S390PCIIOMMU {
     uint64_t pal;
 } S390PCIIOMMU;
 
+typedef struct S390PCIIOMMUTable {
+    uint64_t key;
+    S390PCIIOMMU *iommu[PCI_SLOT_MAX];
+} S390PCIIOMMUTable;
+
 typedef struct S390PCIBusDevice {
     DeviceState qdev;
     PCIDevice *pdev;
@@ -304,7 +309,7 @@ typedef struct S390pciState {
     PCIHostState parent_obj;
     S390PCIBus *bus;
     S390PCIBusDevice *pbdev[PCI_SLOT_MAX];
-    S390PCIIOMMU *iommu[PCI_SLOT_MAX];
+    GHashTable *iommu_table;
     QTAILQ_HEAD(, SeiContainer) pending_sei;
 } S390pciState;
 

From e70377dfa4bbc2e101066ca35675bed4129c5a8c Mon Sep 17 00:00:00 2001
From: Pierre Morel <pmorel@linux.vnet.ibm.com>
Date: Tue, 15 Nov 2016 15:51:38 +0800
Subject: [PATCH 06/12] s390x/pci: change the device array to a list

In order to support a greater number of devices we use a QTAILQ
list of devices instead of a limited array.

This leads us to change:
- every lookup function s390_pci_find_xxx() for QTAILQ
- the FH_MASK_INDEX to index up to 65536 devices

Signed-off-by: Pierre Morel <pmorel@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/s390-pci-bus.c | 100 ++++++++++++++++++++--------------------
 hw/s390x/s390-pci-bus.h |   7 ++-
 2 files changed, 56 insertions(+), 51 deletions(-)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index f3afb17f6a..7ba1ae8b25 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -93,33 +93,24 @@ int chsc_sei_nt2_have_event(void)
 
 S390PCIBusDevice *s390_pci_find_next_avail_dev(S390PCIBusDevice *pbdev)
 {
-    int idx = 0;
-    S390PCIBusDevice *dev = NULL;
     S390pciState *s = s390_get_phb();
+    S390PCIBusDevice *ret = pbdev ? QTAILQ_NEXT(pbdev, link) :
+        QTAILQ_FIRST(&s->zpci_devs);
 
-    if (pbdev) {
-        idx = (pbdev->fh & FH_MASK_INDEX) + 1;
+    while (ret && ret->state == ZPCI_FS_RESERVED) {
+        ret = QTAILQ_NEXT(ret, link);
     }
 
-    for (; idx < PCI_SLOT_MAX; idx++) {
-        dev = s->pbdev[idx];
-        if (dev && dev->state != ZPCI_FS_RESERVED) {
-            return dev;
-        }
-    }
-
-    return NULL;
+    return ret;
 }
 
 S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid)
 {
     S390PCIBusDevice *pbdev;
-    int i;
     S390pciState *s = s390_get_phb();
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        pbdev = s->pbdev[i];
-        if (pbdev && pbdev->fid == fid) {
+    QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
+        if (pbdev->fid == fid) {
             return pbdev;
         }
     }
@@ -203,16 +194,10 @@ out:
 
 static S390PCIBusDevice *s390_pci_find_dev_by_uid(uint16_t uid)
 {
-    int i;
     S390PCIBusDevice *pbdev;
     S390pciState *s = s390_get_phb();
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        pbdev = s->pbdev[i];
-        if (!pbdev) {
-            continue;
-        }
-
+    QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
         if (pbdev->uid == uid) {
             return pbdev;
         }
@@ -223,7 +208,6 @@ static S390PCIBusDevice *s390_pci_find_dev_by_uid(uint16_t uid)
 
 static S390PCIBusDevice *s390_pci_find_dev_by_target(const char *target)
 {
-    int i;
     S390PCIBusDevice *pbdev;
     S390pciState *s = s390_get_phb();
 
@@ -231,12 +215,7 @@ static S390PCIBusDevice *s390_pci_find_dev_by_target(const char *target)
         return NULL;
     }
 
-    for (i = 0; i < PCI_SLOT_MAX; i++) {
-        pbdev = s->pbdev[i];
-        if (!pbdev) {
-            continue;
-        }
-
+    QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
         if (!strcmp(pbdev->target, target)) {
             return pbdev;
         }
@@ -247,9 +226,16 @@ static S390PCIBusDevice *s390_pci_find_dev_by_target(const char *target)
 
 S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx)
 {
+    S390PCIBusDevice *pbdev;
     S390pciState *s = s390_get_phb();
 
-    return s->pbdev[idx & FH_MASK_INDEX];
+    QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
+        if (pbdev->idx == idx) {
+            return pbdev;
+        }
+    }
+
+    return NULL;
 }
 
 S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh)
@@ -257,9 +243,10 @@ S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh)
     S390pciState *s = s390_get_phb();
     S390PCIBusDevice *pbdev;
 
-    pbdev = s->pbdev[fh & FH_MASK_INDEX];
-    if (pbdev && pbdev->fh == fh) {
-        return pbdev;
+    QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
+        if (pbdev->fh == fh) {
+            return pbdev;
+        }
     }
 
     return NULL;
@@ -599,6 +586,7 @@ static int s390_pcihost_init(SysBusDevice *dev)
     s->iommu_table = g_hash_table_new_full(g_int64_hash, g_int64_equal,
                                            NULL, g_free);
     QTAILQ_INIT(&s->pending_sei);
+    QTAILQ_INIT(&s->zpci_devs);
     return 0;
 }
 
@@ -666,6 +654,25 @@ static S390PCIBusDevice *s390_pci_device_new(const char *target)
     return S390_PCI_DEVICE(dev);
 }
 
+static bool s390_pci_alloc_idx(S390PCIBusDevice *pbdev)
+{
+    uint32_t idx;
+    S390pciState *s = s390_get_phb();
+
+    idx = s->next_idx;
+    while (s390_pci_find_dev_by_idx(idx)) {
+        idx = (idx + 1) & FH_MASK_INDEX;
+        if (idx == s->next_idx) {
+            return false;
+        }
+    }
+
+    pbdev->idx = idx;
+    s->next_idx = (idx + 1) & FH_MASK_INDEX;
+
+    return true;
+}
+
 static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
                                   DeviceState *dev, Error **errp)
 {
@@ -713,18 +720,14 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
                                          pbdev->fh, pbdev->fid);
         }
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_S390_PCI_DEVICE)) {
-        int idx;
-
         pbdev = S390_PCI_DEVICE(dev);
-        for (idx = 0; idx < PCI_SLOT_MAX; idx++) {
-            if (!s->pbdev[idx]) {
-                s->pbdev[idx] = pbdev;
-                pbdev->fh = idx;
-                return;
-            }
-        }
 
-        error_setg(errp, "no slot for plugging zpci device");
+        if (!s390_pci_alloc_idx(pbdev)) {
+            error_setg(errp, "no slot for plugging zpci device");
+            return;
+        }
+        pbdev->fh = pbdev->idx;
+        QTAILQ_INSERT_TAIL(&s->zpci_devs, pbdev, link);
     }
 }
 
@@ -750,16 +753,15 @@ static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
 {
     PCIDevice *pci_dev = NULL;
     PCIBus *bus;
-    int32_t devfn, i;
+    int32_t devfn;
     S390PCIBusDevice *pbdev = NULL;
     S390pciState *s = s390_get_phb();
 
     if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
         pci_dev = PCI_DEVICE(dev);
 
-        for (i = 0 ; i < PCI_SLOT_MAX; i++) {
-            if (s->pbdev[i] && s->pbdev[i]->pdev == pci_dev) {
-                pbdev = s->pbdev[i];
+        QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
+            if (pbdev->pdev == pci_dev) {
                 break;
             }
         }
@@ -802,7 +804,7 @@ static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
     pbdev->state = ZPCI_FS_RESERVED;
 out:
     pbdev->fid = 0;
-    s->pbdev[pbdev->fh & FH_MASK_INDEX] = NULL;
+    QTAILQ_REMOVE(&s->zpci_devs, pbdev, link);
     object_unparent(OBJECT(pbdev));
 }
 
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index fe108e9330..1bd37ad821 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -27,7 +27,7 @@
 #define FH_MASK_ENABLE   0x80000000
 #define FH_MASK_INSTANCE 0x7f000000
 #define FH_MASK_SHM      0x00ff0000
-#define FH_MASK_INDEX    0x0000001f
+#define FH_MASK_INDEX    0x0000ffff
 #define FH_SHM_VFIO      0x00010000
 #define FH_SHM_EMUL      0x00020000
 #define S390_PCIPT_ADAPTER 2
@@ -285,6 +285,7 @@ typedef struct S390PCIBusDevice {
     ZpciState state;
     char *target;
     uint16_t uid;
+    uint32_t idx;
     uint32_t fh;
     uint32_t fid;
     bool fid_defined;
@@ -299,6 +300,7 @@ typedef struct S390PCIBusDevice {
     IndAddr *summary_ind;
     IndAddr *indicator;
     QEMUTimer *release_timer;
+    QTAILQ_ENTRY(S390PCIBusDevice) link;
 } S390PCIBusDevice;
 
 typedef struct S390PCIBus {
@@ -307,10 +309,11 @@ typedef struct S390PCIBus {
 
 typedef struct S390pciState {
     PCIHostState parent_obj;
+    uint32_t next_idx;
     S390PCIBus *bus;
-    S390PCIBusDevice *pbdev[PCI_SLOT_MAX];
     GHashTable *iommu_table;
     QTAILQ_HEAD(, SeiContainer) pending_sei;
+    QTAILQ_HEAD(, S390PCIBusDevice) zpci_devs;
 } S390pciState;
 
 int chsc_sei_nt2_get_event(void *res);

From a975a24aed372ba665a8b159e4cd14b6e105a296 Mon Sep 17 00:00:00 2001
From: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Date: Wed, 23 Nov 2016 11:08:29 +0800
Subject: [PATCH 07/12] s390x/pci: optimize calling s390_get_phb()

A function may recursively call device search functions or may call
serveral different device search function. Passing the S390pciState to
search functions as an argument instead of looking up it inside the
search functions lowers the number of calling s390_get_phb().

Signed-off-by: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Reviewed-by: Pierre Morel <pmorel@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/s390-pci-bus.c  | 66 +++++++++++++++++++---------------------
 hw/s390x/s390-pci-bus.h  | 10 +++---
 hw/s390x/s390-pci-inst.c | 24 ++++++++-------
 target/s390x/kvm.c       |  2 +-
 4 files changed, 52 insertions(+), 50 deletions(-)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 7ba1ae8b25..70673af591 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -31,7 +31,7 @@
     do { } while (0)
 #endif
 
-static S390pciState *s390_get_phb(void)
+S390pciState *s390_get_phb(void)
 {
     static S390pciState *phb;
 
@@ -91,9 +91,9 @@ int chsc_sei_nt2_have_event(void)
     return !QTAILQ_EMPTY(&s->pending_sei);
 }
 
-S390PCIBusDevice *s390_pci_find_next_avail_dev(S390PCIBusDevice *pbdev)
+S390PCIBusDevice *s390_pci_find_next_avail_dev(S390pciState *s,
+                                               S390PCIBusDevice *pbdev)
 {
-    S390pciState *s = s390_get_phb();
     S390PCIBusDevice *ret = pbdev ? QTAILQ_NEXT(pbdev, link) :
         QTAILQ_FIRST(&s->zpci_devs);
 
@@ -104,10 +104,9 @@ S390PCIBusDevice *s390_pci_find_next_avail_dev(S390PCIBusDevice *pbdev)
     return ret;
 }
 
-S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid)
+S390PCIBusDevice *s390_pci_find_dev_by_fid(S390pciState *s, uint32_t fid)
 {
     S390PCIBusDevice *pbdev;
-    S390pciState *s = s390_get_phb();
 
     QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
         if (pbdev->fid == fid) {
@@ -121,7 +120,8 @@ S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid)
 void s390_pci_sclp_configure(SCCB *sccb)
 {
     PciCfgSccb *psccb = (PciCfgSccb *)sccb;
-    S390PCIBusDevice *pbdev = s390_pci_find_dev_by_fid(be32_to_cpu(psccb->aid));
+    S390PCIBusDevice *pbdev = s390_pci_find_dev_by_fid(s390_get_phb(),
+                                                       be32_to_cpu(psccb->aid));
     uint16_t rc;
 
     if (be16_to_cpu(sccb->h.length) < 16) {
@@ -153,7 +153,8 @@ out:
 void s390_pci_sclp_deconfigure(SCCB *sccb)
 {
     PciCfgSccb *psccb = (PciCfgSccb *)sccb;
-    S390PCIBusDevice *pbdev = s390_pci_find_dev_by_fid(be32_to_cpu(psccb->aid));
+    S390PCIBusDevice *pbdev = s390_pci_find_dev_by_fid(s390_get_phb(),
+                                                       be32_to_cpu(psccb->aid));
     uint16_t rc;
 
     if (be16_to_cpu(sccb->h.length) < 16) {
@@ -192,10 +193,9 @@ out:
     psccb->header.response_code = cpu_to_be16(rc);
 }
 
-static S390PCIBusDevice *s390_pci_find_dev_by_uid(uint16_t uid)
+static S390PCIBusDevice *s390_pci_find_dev_by_uid(S390pciState *s, uint16_t uid)
 {
     S390PCIBusDevice *pbdev;
-    S390pciState *s = s390_get_phb();
 
     QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
         if (pbdev->uid == uid) {
@@ -206,10 +206,10 @@ static S390PCIBusDevice *s390_pci_find_dev_by_uid(uint16_t uid)
     return NULL;
 }
 
-static S390PCIBusDevice *s390_pci_find_dev_by_target(const char *target)
+static S390PCIBusDevice *s390_pci_find_dev_by_target(S390pciState *s,
+                                                     const char *target)
 {
     S390PCIBusDevice *pbdev;
-    S390pciState *s = s390_get_phb();
 
     if (!target) {
         return NULL;
@@ -224,10 +224,9 @@ static S390PCIBusDevice *s390_pci_find_dev_by_target(const char *target)
     return NULL;
 }
 
-S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx)
+S390PCIBusDevice *s390_pci_find_dev_by_idx(S390pciState *s, uint32_t idx)
 {
     S390PCIBusDevice *pbdev;
-    S390pciState *s = s390_get_phb();
 
     QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
         if (pbdev->idx == idx) {
@@ -238,9 +237,8 @@ S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx)
     return NULL;
 }
 
-S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh)
+S390PCIBusDevice *s390_pci_find_dev_by_fh(S390pciState *s, uint32_t fh)
 {
-    S390pciState *s = s390_get_phb();
     S390PCIBusDevice *pbdev;
 
     QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
@@ -544,7 +542,7 @@ void s390_pci_iommu_disable(S390PCIIOMMU *iommu)
     object_unparent(OBJECT(&iommu->iommu_mr));
 }
 
-static void s390_pci_iommu_free(PCIBus *bus, int32_t devfn)
+static void s390_pci_iommu_free(S390pciState *s, PCIBus *bus, int32_t devfn)
 {
     uint64_t key = (uintptr_t)bus;
     S390PCIIOMMUTable *table = g_hash_table_lookup(s->iommu_table, &key);
@@ -638,10 +636,10 @@ static void s390_pci_msix_free(S390PCIBusDevice *pbdev)
     object_unparent(OBJECT(&pbdev->msix_notify_mr));
 }
 
-static S390PCIBusDevice *s390_pci_device_new(const char *target)
+static S390PCIBusDevice *s390_pci_device_new(S390pciState *s,
+                                             const char *target)
 {
     DeviceState *dev = NULL;
-    S390pciState *s = s390_get_phb();
 
     dev = qdev_try_create(BUS(s->bus), TYPE_S390_PCI_DEVICE);
     if (!dev) {
@@ -654,13 +652,12 @@ static S390PCIBusDevice *s390_pci_device_new(const char *target)
     return S390_PCI_DEVICE(dev);
 }
 
-static bool s390_pci_alloc_idx(S390PCIBusDevice *pbdev)
+static bool s390_pci_alloc_idx(S390pciState *s, S390PCIBusDevice *pbdev)
 {
     uint32_t idx;
-    S390pciState *s = s390_get_phb();
 
     idx = s->next_idx;
-    while (s390_pci_find_dev_by_idx(idx)) {
+    while (s390_pci_find_dev_by_idx(s, idx)) {
         idx = (idx + 1) & FH_MASK_INDEX;
         if (idx == s->next_idx) {
             return false;
@@ -692,9 +689,9 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
                                       PCI_FUNC(pdev->devfn));
         }
 
-        pbdev = s390_pci_find_dev_by_target(dev->id);
+        pbdev = s390_pci_find_dev_by_target(s, dev->id);
         if (!pbdev) {
-            pbdev = s390_pci_device_new(dev->id);
+            pbdev = s390_pci_device_new(s, dev->id);
             if (!pbdev) {
                 error_setg(errp, "create zpci device failed");
                 return;
@@ -722,7 +719,7 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_S390_PCI_DEVICE)) {
         pbdev = S390_PCI_DEVICE(dev);
 
-        if (!s390_pci_alloc_idx(pbdev)) {
+        if (!s390_pci_alloc_idx(s, pbdev)) {
             error_setg(errp, "no slot for plugging zpci device");
             return;
         }
@@ -799,7 +796,7 @@ static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
     devfn = pci_dev->devfn;
     object_unparent(OBJECT(pci_dev));
     s390_pci_msix_free(pbdev);
-    s390_pci_iommu_free(bus, devfn);
+    s390_pci_iommu_free(s, bus, devfn);
     pbdev->pdev = NULL;
     pbdev->state = ZPCI_FS_RESERVED;
 out:
@@ -838,13 +835,13 @@ static const TypeInfo s390_pcibus_info = {
     .instance_size = sizeof(S390PCIBus),
 };
 
-static uint16_t s390_pci_generate_uid(void)
+static uint16_t s390_pci_generate_uid(S390pciState *s)
 {
     uint16_t uid = 0;
 
     do {
         uid++;
-        if (!s390_pci_find_dev_by_uid(uid)) {
+        if (!s390_pci_find_dev_by_uid(s, uid)) {
             return uid;
         }
     } while (uid < ZPCI_MAX_UID);
@@ -852,12 +849,12 @@ static uint16_t s390_pci_generate_uid(void)
     return UID_UNDEFINED;
 }
 
-static uint32_t s390_pci_generate_fid(Error **errp)
+static uint32_t s390_pci_generate_fid(S390pciState *s, Error **errp)
 {
     uint32_t fid = 0;
 
     do {
-        if (!s390_pci_find_dev_by_fid(fid)) {
+        if (!s390_pci_find_dev_by_fid(s, fid)) {
             return fid;
         }
     } while (fid++ != ZPCI_MAX_FID);
@@ -869,25 +866,26 @@ static uint32_t s390_pci_generate_fid(Error **errp)
 static void s390_pci_device_realize(DeviceState *dev, Error **errp)
 {
     S390PCIBusDevice *zpci = S390_PCI_DEVICE(dev);
+    S390pciState *s = s390_get_phb();
 
     if (!zpci->target) {
         error_setg(errp, "target must be defined");
         return;
     }
 
-    if (s390_pci_find_dev_by_target(zpci->target)) {
+    if (s390_pci_find_dev_by_target(s, zpci->target)) {
         error_setg(errp, "target %s already has an associated zpci device",
                    zpci->target);
         return;
     }
 
     if (zpci->uid == UID_UNDEFINED) {
-        zpci->uid = s390_pci_generate_uid();
+        zpci->uid = s390_pci_generate_uid(s);
         if (!zpci->uid) {
             error_setg(errp, "no free uid could be found");
             return;
         }
-    } else if (s390_pci_find_dev_by_uid(zpci->uid)) {
+    } else if (s390_pci_find_dev_by_uid(s, zpci->uid)) {
         error_setg(errp, "uid %u already in use", zpci->uid);
         return;
     }
@@ -895,12 +893,12 @@ static void s390_pci_device_realize(DeviceState *dev, Error **errp)
     if (!zpci->fid_defined) {
         Error *local_error = NULL;
 
-        zpci->fid = s390_pci_generate_fid(&local_error);
+        zpci->fid = s390_pci_generate_fid(s, &local_error);
         if (local_error) {
             error_propagate(errp, local_error);
             return;
         }
-    } else if (s390_pci_find_dev_by_fid(zpci->fid)) {
+    } else if (s390_pci_find_dev_by_fid(s, zpci->fid)) {
         error_setg(errp, "fid %u already in use", zpci->fid);
         return;
     }
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index 1bd37ad821..fbdc64febf 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -316,6 +316,7 @@ typedef struct S390pciState {
     QTAILQ_HEAD(, S390PCIBusDevice) zpci_devs;
 } S390pciState;
 
+S390pciState *s390_get_phb(void);
 int chsc_sei_nt2_get_event(void *res);
 int chsc_sei_nt2_have_event(void);
 void s390_pci_sclp_configure(SCCB *sccb);
@@ -324,9 +325,10 @@ void s390_pci_iommu_enable(S390PCIIOMMU *iommu);
 void s390_pci_iommu_disable(S390PCIIOMMU *iommu);
 void s390_pci_generate_error_event(uint16_t pec, uint32_t fh, uint32_t fid,
                                    uint64_t faddr, uint32_t e);
-S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx);
-S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh);
-S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid);
-S390PCIBusDevice *s390_pci_find_next_avail_dev(S390PCIBusDevice *pbdev);
+S390PCIBusDevice *s390_pci_find_dev_by_idx(S390pciState *s, uint32_t idx);
+S390PCIBusDevice *s390_pci_find_dev_by_fh(S390pciState *s, uint32_t fh);
+S390PCIBusDevice *s390_pci_find_dev_by_fid(S390pciState *s, uint32_t fid);
+S390PCIBusDevice *s390_pci_find_next_avail_dev(S390pciState *s,
+                                               S390PCIBusDevice *pbdev);
 
 #endif
diff --git a/hw/s390x/s390-pci-inst.c b/hw/s390x/s390-pci-inst.c
index 7dbdb0c6d6..405cccd2d0 100644
--- a/hw/s390x/s390-pci-inst.c
+++ b/hw/s390x/s390-pci-inst.c
@@ -38,6 +38,7 @@ static void s390_set_status_code(CPUS390XState *env,
 static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
 {
     S390PCIBusDevice *pbdev = NULL;
+    S390pciState *s = s390_get_phb();
     uint32_t res_code, initial_l2, g_l2;
     int rc, i;
     uint64_t resume_token;
@@ -65,14 +66,14 @@ static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
     resume_token = ldq_p(&rrb->request.resume_token);
 
     if (resume_token) {
-        pbdev = s390_pci_find_dev_by_idx(resume_token);
+        pbdev = s390_pci_find_dev_by_idx(s, resume_token);
         if (!pbdev) {
             res_code = CLP_RC_LISTPCI_BADRT;
             rc = -EINVAL;
             goto out;
         }
     } else {
-        pbdev = s390_pci_find_next_avail_dev(NULL);
+        pbdev = s390_pci_find_next_avail_dev(s, NULL);
     }
 
     if (lduw_p(&rrb->response.hdr.len) < 48) {
@@ -118,7 +119,7 @@ static int list_pci(ClpReqRspListPci *rrb, uint8_t *cc)
                 lduw_p(&rrb->response.fh_list[i].device_id),
                 ldl_p(&rrb->response.fh_list[i].fid),
                 ldl_p(&rrb->response.fh_list[i].fh));
-        pbdev = s390_pci_find_next_avail_dev(pbdev);
+        pbdev = s390_pci_find_next_avail_dev(s, pbdev);
         i++;
     }
 
@@ -148,6 +149,7 @@ int clp_service_call(S390CPU *cpu, uint8_t r2)
     uint8_t buffer[4096 * 2];
     uint8_t cc = 0;
     CPUS390XState *env = &cpu->env;
+    S390pciState *s = s390_get_phb();
     int i;
 
     cpu_synchronize_state(CPU(cpu));
@@ -202,7 +204,7 @@ int clp_service_call(S390CPU *cpu, uint8_t r2)
         ClpReqSetPci *reqsetpci = (ClpReqSetPci *)reqh;
         ClpRspSetPci *ressetpci = (ClpRspSetPci *)resh;
 
-        pbdev = s390_pci_find_dev_by_fh(ldl_p(&reqsetpci->fh));
+        pbdev = s390_pci_find_dev_by_fh(s, ldl_p(&reqsetpci->fh));
         if (!pbdev) {
                 stw_p(&ressetpci->hdr.rsp, CLP_RC_SETPCIFN_FH);
                 goto out;
@@ -253,7 +255,7 @@ int clp_service_call(S390CPU *cpu, uint8_t r2)
         ClpReqQueryPci *reqquery = (ClpReqQueryPci *)reqh;
         ClpRspQueryPci *resquery = (ClpRspQueryPci *)resh;
 
-        pbdev = s390_pci_find_dev_by_fh(ldl_p(&reqquery->fh));
+        pbdev = s390_pci_find_dev_by_fh(s, ldl_p(&reqquery->fh));
         if (!pbdev) {
             DPRINTF("query pci no pci dev\n");
             stw_p(&resquery->hdr.rsp, CLP_RC_SETPCIFN_FH);
@@ -338,7 +340,7 @@ int pcilg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
     len = env->regs[r2] & 0xf;
     offset = env->regs[r2 + 1];
 
-    pbdev = s390_pci_find_dev_by_fh(fh);
+    pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh);
     if (!pbdev) {
         DPRINTF("pcilg no pci dev\n");
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
@@ -471,7 +473,7 @@ int pcistg_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
     len = env->regs[r2] & 0xf;
     offset = env->regs[r2 + 1];
 
-    pbdev = s390_pci_find_dev_by_fh(fh);
+    pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh);
     if (!pbdev) {
         DPRINTF("pcistg no pci dev\n");
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
@@ -576,7 +578,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
     start = env->regs[r2];
     end = start + env->regs[r2 + 1];
 
-    pbdev = s390_pci_find_dev_by_fh(fh);
+    pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh);
     if (!pbdev) {
         DPRINTF("rpcit no pci dev\n");
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
@@ -679,7 +681,7 @@ int pcistb_service_call(S390CPU *cpu, uint8_t r1, uint8_t r3, uint64_t gaddr,
         return 0;
     }
 
-    pbdev = s390_pci_find_dev_by_fh(fh);
+    pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh);
     if (!pbdev) {
         DPRINTF("pcistb no pci dev fh 0x%x\n", fh);
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
@@ -845,7 +847,7 @@ int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
         return 0;
     }
 
-    pbdev = s390_pci_find_dev_by_fh(fh);
+    pbdev = s390_pci_find_dev_by_fh(s390_get_phb(), fh);
     if (!pbdev) {
         DPRINTF("mpcifc no pci dev fh 0x%x\n", fh);
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
@@ -990,7 +992,7 @@ int stpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)
         return 0;
     }
 
-    pbdev = s390_pci_find_dev_by_idx(fh & FH_MASK_INDEX);
+    pbdev = s390_pci_find_dev_by_idx(s390_get_phb(), fh & FH_MASK_INDEX);
     if (!pbdev) {
         setcc(cpu, ZPCI_PCI_LS_INVAL_HANDLE);
         return 0;
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 97afe02599..c47db9a884 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -2301,7 +2301,7 @@ int kvm_arch_fixup_msi_route(struct kvm_irq_routing_entry *route,
     uint32_t idx = data >> ZPCI_MSI_VEC_BITS;
     uint32_t vec = data & ZPCI_MSI_VEC_MASK;
 
-    pbdev = s390_pci_find_dev_by_idx(idx);
+    pbdev = s390_pci_find_dev_by_idx(s390_get_phb(), idx);
     if (!pbdev) {
         DPRINTF("add_msi_route no dev\n");
         return -ENODEV;

From 3fc92a24f333b48652a2373c296f641c04b08bae Mon Sep 17 00:00:00 2001
From: Pierre Morel <pmorel@linux.vnet.ibm.com>
Date: Mon, 24 Oct 2016 14:50:35 +0200
Subject: [PATCH 08/12] s390x/pci: PCI multibus bridge handling

When the hotplug handler detects a PCI bridge, the secondary bus has
been initialized by the core PCI code. We give the secondary bus the
bridge name and associate to it the IOMMU handling and
hotplug/hotunplug callbacks.

Signed-off-by: Pierre Morel <pmorel@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/s390-pci-bus.c | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 70673af591..ce29320592 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -19,6 +19,7 @@
 #include "s390-pci-bus.h"
 #include "s390-pci-inst.h"
 #include "hw/pci/pci_bus.h"
+#include "hw/pci/pci_bridge.h"
 #include "hw/pci/msi.h"
 #include "qemu/error-report.h"
 
@@ -677,7 +678,16 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
     S390PCIBusDevice *pbdev = NULL;
     S390pciState *s = s390_get_phb();
 
-    if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_BRIDGE)) {
+        BusState *bus;
+        PCIBridge *pb = PCI_BRIDGE(dev);
+
+        pci_bridge_map_irq(pb, dev->id, s390_pci_map_irq);
+        pci_setup_iommu(&pb->sec_bus, s390_pci_dma_iommu, s);
+
+        bus = BUS(&pb->sec_bus);
+        qbus_set_hotplug_handler(bus, DEVICE(s), errp);
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
         pdev = PCI_DEVICE(dev);
 
         if (!dev->id) {
@@ -754,7 +764,10 @@ static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
     S390PCIBusDevice *pbdev = NULL;
     S390pciState *s = s390_get_phb();
 
-    if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
+    if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_BRIDGE)) {
+        error_setg(errp, "PCI bridge hot unplug currently not supported");
+        return;
+    } else if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
         pci_dev = PCI_DEVICE(dev);
 
         QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {

From df8dd91b99ed7d27c1cf9a0e6283b2452192d9a1 Mon Sep 17 00:00:00 2001
From: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Date: Wed, 9 Nov 2016 21:30:50 +0800
Subject: [PATCH 09/12] s390x/pci: use hashtable to look up zpci via fh

After PCI multibus is supported, more than 32 PCI devices could be
plugged. The current implementation of s390_pci_find_dev_by_fh()
appears low performance if there's a huge number of PCI devices
plugged. Therefore we introduce a hashtable using idx as key to store
zpci device's pointer on account of translating fh to idx very easily.

Signed-off-by: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Reviewed-by: Pierre Morel <pmorel@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/s390-pci-bus.c | 22 ++++++++--------------
 hw/s390x/s390-pci-bus.h |  1 +
 2 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index ce29320592..c2ae01c9a6 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -227,25 +227,16 @@ static S390PCIBusDevice *s390_pci_find_dev_by_target(S390pciState *s,
 
 S390PCIBusDevice *s390_pci_find_dev_by_idx(S390pciState *s, uint32_t idx)
 {
-    S390PCIBusDevice *pbdev;
-
-    QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
-        if (pbdev->idx == idx) {
-            return pbdev;
-        }
-    }
-
-    return NULL;
+    return g_hash_table_lookup(s->zpci_table, &idx);
 }
 
 S390PCIBusDevice *s390_pci_find_dev_by_fh(S390pciState *s, uint32_t fh)
 {
-    S390PCIBusDevice *pbdev;
+    uint32_t idx = FH_MASK_INDEX & fh;
+    S390PCIBusDevice *pbdev = s390_pci_find_dev_by_idx(s, idx);
 
-    QTAILQ_FOREACH(pbdev, &s->zpci_devs, link) {
-        if (pbdev->fh == fh) {
-            return pbdev;
-        }
+    if (pbdev && pbdev->fh == fh) {
+        return pbdev;
     }
 
     return NULL;
@@ -584,6 +575,7 @@ static int s390_pcihost_init(SysBusDevice *dev)
 
     s->iommu_table = g_hash_table_new_full(g_int64_hash, g_int64_equal,
                                            NULL, g_free);
+    s->zpci_table = g_hash_table_new_full(g_int_hash, g_int_equal, NULL, NULL);
     QTAILQ_INIT(&s->pending_sei);
     QTAILQ_INIT(&s->zpci_devs);
     return 0;
@@ -735,6 +727,7 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
         }
         pbdev->fh = pbdev->idx;
         QTAILQ_INSERT_TAIL(&s->zpci_devs, pbdev, link);
+        g_hash_table_insert(s->zpci_table, &pbdev->idx, pbdev);
     }
 }
 
@@ -815,6 +808,7 @@ static void s390_pcihost_hot_unplug(HotplugHandler *hotplug_dev,
 out:
     pbdev->fid = 0;
     QTAILQ_REMOVE(&s->zpci_devs, pbdev, link);
+    g_hash_table_remove(s->zpci_table, &pbdev->idx);
     object_unparent(OBJECT(pbdev));
 }
 
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index fbdc64febf..b82b18eb07 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -312,6 +312,7 @@ typedef struct S390pciState {
     uint32_t next_idx;
     S390PCIBus *bus;
     GHashTable *iommu_table;
+    GHashTable *zpci_table;
     QTAILQ_HEAD(, SeiContainer) pending_sei;
     QTAILQ_HEAD(, S390PCIBusDevice) zpci_devs;
 } S390pciState;

From d2f07120a35ab31715e9c73438705b635873d06a Mon Sep 17 00:00:00 2001
From: Pierre Morel <pmorel@linux.vnet.ibm.com>
Date: Wed, 23 Nov 2016 14:26:34 +0800
Subject: [PATCH 10/12] s390x/pci: handle PCIBridge bus number

The PCI bus number is usually set by the host during the enumeration.

In the s390 architecture we neither get a Device Tree nor have an
enumeration understanding bridge devices.

Let's fake the enumeration on reset and set the PCI_PRIMARY_BUS,
PCI_SECONDARY_BUS and PCI_SUBORDINATE_BUS config entries for the
bridges.

Let's add the configuration of these three config entries on bridge hot
plug.

The bus number is calculated based on a new entry, bus_num of the
S390pciState device.

This commit is inspired by what spapr pci does.

Signed-off-by: Pierre Morel <pmorel@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/s390-pci-bus.c | 52 +++++++++++++++++++++++++++++++++++++++++
 hw/s390x/s390-pci-bus.h |  1 +
 2 files changed, 53 insertions(+)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index c2ae01c9a6..5cf97b4f7d 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -576,6 +576,7 @@ static int s390_pcihost_init(SysBusDevice *dev)
     s->iommu_table = g_hash_table_new_full(g_int64_hash, g_int64_equal,
                                            NULL, g_free);
     s->zpci_table = g_hash_table_new_full(g_int_hash, g_int_equal, NULL, NULL);
+    s->bus_no = 0;
     QTAILQ_INIT(&s->pending_sei);
     QTAILQ_INIT(&s->zpci_devs);
     return 0;
@@ -673,12 +674,24 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
     if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_BRIDGE)) {
         BusState *bus;
         PCIBridge *pb = PCI_BRIDGE(dev);
+        PCIDevice *pdev = PCI_DEVICE(dev);
 
         pci_bridge_map_irq(pb, dev->id, s390_pci_map_irq);
         pci_setup_iommu(&pb->sec_bus, s390_pci_dma_iommu, s);
 
         bus = BUS(&pb->sec_bus);
         qbus_set_hotplug_handler(bus, DEVICE(s), errp);
+
+        if (dev->hotplugged) {
+            pci_default_write_config(pdev, PCI_PRIMARY_BUS, s->bus_no, 1);
+            s->bus_no += 1;
+            pci_default_write_config(pdev, PCI_SECONDARY_BUS, s->bus_no, 1);
+            do {
+                pdev = pdev->bus->parent_dev;
+                pci_default_write_config(pdev, PCI_SUBORDINATE_BUS,
+                                         s->bus_no, 1);
+            } while (pdev->bus && pci_bus_num(pdev->bus));
+        }
     } else if (object_dynamic_cast(OBJECT(dev), TYPE_PCI_DEVICE)) {
         pdev = PCI_DEVICE(dev);
 
@@ -812,6 +825,44 @@ out:
     object_unparent(OBJECT(pbdev));
 }
 
+static void s390_pci_enumerate_bridge(PCIBus *bus, PCIDevice *pdev,
+                                      void *opaque)
+{
+    S390pciState *s = opaque;
+    unsigned int primary = s->bus_no;
+    unsigned int subordinate = 0xff;
+    PCIBus *sec_bus = NULL;
+
+    if ((pci_default_read_config(pdev, PCI_HEADER_TYPE, 1) !=
+         PCI_HEADER_TYPE_BRIDGE)) {
+        return;
+    }
+
+    (s->bus_no)++;
+    pci_default_write_config(pdev, PCI_PRIMARY_BUS, primary, 1);
+    pci_default_write_config(pdev, PCI_SECONDARY_BUS, s->bus_no, 1);
+    pci_default_write_config(pdev, PCI_SUBORDINATE_BUS, s->bus_no, 1);
+
+    sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(pdev));
+    if (!sec_bus) {
+        return;
+    }
+
+    pci_default_write_config(pdev, PCI_SUBORDINATE_BUS, subordinate, 1);
+    pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
+                        s390_pci_enumerate_bridge, s);
+    pci_default_write_config(pdev, PCI_SUBORDINATE_BUS, s->bus_no, 1);
+}
+
+static void s390_pcihost_reset(DeviceState *dev)
+{
+    S390pciState *s = S390_PCI_HOST_BRIDGE(dev);
+    PCIBus *bus = s->parent_obj.bus;
+
+    s->bus_no = 0;
+    pci_for_each_device(bus, pci_bus_num(bus), s390_pci_enumerate_bridge, s);
+}
+
 static void s390_pcihost_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
@@ -819,6 +870,7 @@ static void s390_pcihost_class_init(ObjectClass *klass, void *data)
     HotplugHandlerClass *hc = HOTPLUG_HANDLER_CLASS(klass);
 
     dc->cannot_instantiate_with_device_add_yet = true;
+    dc->reset = s390_pcihost_reset;
     k->init = s390_pcihost_init;
     hc->plug = s390_pcihost_hot_plug;
     hc->unplug = s390_pcihost_hot_unplug;
diff --git a/hw/s390x/s390-pci-bus.h b/hw/s390x/s390-pci-bus.h
index b82b18eb07..b0adefa788 100644
--- a/hw/s390x/s390-pci-bus.h
+++ b/hw/s390x/s390-pci-bus.h
@@ -310,6 +310,7 @@ typedef struct S390PCIBus {
 typedef struct S390pciState {
     PCIHostState parent_obj;
     uint32_t next_idx;
+    int bus_no;
     S390PCIBus *bus;
     GHashTable *iommu_table;
     GHashTable *zpci_table;

From 857cc71985dcd553a601fb9a0df7919359c23115 Mon Sep 17 00:00:00 2001
From: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Date: Tue, 13 Dec 2016 18:31:25 +0800
Subject: [PATCH 11/12] s390x/pci: merge msix init functions

Currently there're two functions, s390_pci_setup_msix() and
s390_pci_msix_init(), for msix initialization, and being called once
for each zpci device plugging. Let's integrate them.

Moreover msix is mandatory in s390 architecture. So we ensure the pci
device being plugged supports msix. For vfio (which is the only tested
setup so far), nothing changes.

Signed-off-by: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Reviewed-by: Pierre Morel <pmorel@linux.vnet.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/s390-pci-bus.c | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/hw/s390x/s390-pci-bus.c b/hw/s390x/s390-pci-bus.c
index 5cf97b4f7d..69b0291e8a 100644
--- a/hw/s390x/s390-pci-bus.c
+++ b/hw/s390x/s390-pci-bus.c
@@ -582,8 +582,9 @@ static int s390_pcihost_init(SysBusDevice *dev)
     return 0;
 }
 
-static int s390_pci_setup_msix(S390PCIBusDevice *pbdev)
+static int s390_pci_msix_init(S390PCIBusDevice *pbdev)
 {
+    char *name;
     uint8_t pos;
     uint16_t ctrl;
     uint32_t table, pba;
@@ -591,7 +592,7 @@ static int s390_pci_setup_msix(S390PCIBusDevice *pbdev)
     pos = pci_find_capability(pbdev->pdev, PCI_CAP_ID_MSIX);
     if (!pos) {
         pbdev->msix.available = false;
-        return 0;
+        return -1;
     }
 
     ctrl = pci_host_config_read_common(pbdev->pdev, pos + PCI_MSIX_FLAGS,
@@ -607,21 +608,15 @@ static int s390_pci_setup_msix(S390PCIBusDevice *pbdev)
     pbdev->msix.pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
     pbdev->msix.entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
     pbdev->msix.available = true;
-    return 0;
-}
-
-static void s390_pci_msix_init(S390PCIBusDevice *pbdev)
-{
-    char *name;
 
     name = g_strdup_printf("msix-s390-%04x", pbdev->uid);
-
     memory_region_init_io(&pbdev->msix_notify_mr, OBJECT(pbdev),
                           &s390_msi_ctrl_ops, pbdev, name, PAGE_SIZE);
     memory_region_add_subregion(&pbdev->iommu->mr, ZPCI_MSI_ADDR,
                                 &pbdev->msix_notify_mr);
-
     g_free(name);
+
+    return 0;
 }
 
 static void s390_pci_msix_free(S390PCIBusDevice *pbdev)
@@ -724,8 +719,11 @@ static void s390_pcihost_hot_plug(HotplugHandler *hotplug_dev,
         pbdev->iommu->pbdev = pbdev;
         pbdev->state = ZPCI_FS_STANDBY;
 
-        s390_pci_msix_init(pbdev);
-        s390_pci_setup_msix(pbdev);
+        if (s390_pci_msix_init(pbdev)) {
+            error_setg(errp, "MSI-X support is mandatory "
+                       "in the S390 architecture");
+            return;
+        }
 
         if (dev->hotplugged) {
             s390_pci_generate_plug_event(HP_EVENT_RESERVED_TO_STANDBY,

From 8c797e758ad41b8dd6ea6f72c6f0194dedb30a2f Mon Sep 17 00:00:00 2001
From: "Michael S. Tsirkin" <mst@redhat.com>
Date: Thu, 12 Jan 2017 23:26:22 +0200
Subject: [PATCH 12/12] virtio-ccw: fix ring sizing

Current code seems to assume ring size is
always decreased but this is not required by spec:
what spec says is just that size can not exceed
the maximum. Fix it up.

Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Message-Id: <1484256243-1982-1-git-send-email-mst@redhat.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>
---
 hw/s390x/virtio-ccw.c      | 2 +-
 hw/virtio/virtio.c         | 5 +++++
 include/hw/virtio/virtio.h | 1 +
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index 07650683f7..63c46373fb 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -149,7 +149,7 @@ static int virtio_ccw_set_vqs(SubchDev *sch, VqInfoBlock *info,
     } else {
         if (info) {
             /* virtio-1 allows changing the ring size. */
-            if (virtio_queue_get_num(vdev, index) < num) {
+            if (virtio_queue_get_max_num(vdev, index) < num) {
                 /* Fail if we exceed the maximum number. */
                 return -EINVAL;
             }
diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c
index aa4f38f50a..94d7dbae6c 100644
--- a/hw/virtio/virtio.c
+++ b/hw/virtio/virtio.c
@@ -1262,6 +1262,11 @@ int virtio_queue_get_num(VirtIODevice *vdev, int n)
     return vdev->vq[n].vring.num;
 }
 
+int virtio_queue_get_max_num(VirtIODevice *vdev, int n)
+{
+    return vdev->vq[n].vring.num_default;
+}
+
 int virtio_get_num_queues(VirtIODevice *vdev)
 {
     int i;
diff --git a/include/hw/virtio/virtio.h b/include/hw/virtio/virtio.h
index e5541c61f7..6523bacd2f 100644
--- a/include/hw/virtio/virtio.h
+++ b/include/hw/virtio/virtio.h
@@ -228,6 +228,7 @@ void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr);
 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n);
 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num);
 int virtio_queue_get_num(VirtIODevice *vdev, int n);
+int virtio_queue_get_max_num(VirtIODevice *vdev, int n);
 int virtio_get_num_queues(VirtIODevice *vdev);
 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
                             hwaddr avail, hwaddr used);