#============================================================================ # This library is free software; you can redistribute it and/or # modify it under the terms of version 2.1 of the GNU Lesser General Public # License as published by the Free Software Foundation. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #============================================================================ # Copyright (C) 2004, 2005 Mike Wray # Copyright (C) 2005 XenSource Ltd #============================================================================ import types import time from xen.xend import sxp from xen.xend import XendOptions xoptions = XendOptions.instance() from xen.xend import arch from xen.xend.XendError import VmError from xen.xend.XendLogging import log from xen.xend.XendConstants import * from xen.xend.server.DevController import DevController from xen.xend.server.DevConstants import xenbusState import xen.lowlevel.xc from xen.util.pci import * import resource import re from xen.xend.server.pciquirk import * from xen.xend.xenstore.xstransact import xstransact from xen.xend.xenstore.xswatch import xswatch xc = xen.lowlevel.xc.xc() #Calculate PAGE_SHIFT: number of bits to shift an address to get the page number PAGE_SIZE = resource.getpagesize() PAGE_SHIFT = 0 t = PAGE_SIZE while not (t&1): t>>=1 PAGE_SHIFT+=1 def parse_hex(val): try: if isinstance(val, types.StringTypes): return int(val, 16) else: return val except ValueError: return None def get_assigned_pci_devices(domid): dev_str_list = [] path = '/local/domain/0/backend/pci/%u/0/' % domid num_devs = xstransact.Read(path + 'num_devs'); if num_devs is None or num_devs == "": return dev_str_list num_devs = int(num_devs) for i in range(num_devs): dev_str = xstransact.Read(path + 'dev-%i' % i) dev_str_list = dev_str_list + [dev_str] return dev_str_list def get_all_assigned_pci_devices(domid = 0): dom_list = xstransact.List('/local/domain') pci_str_list = [] ti = 0 ts = xstransact.Read('/local/domain/' + str(domid) + '/target') if ts is not None : ti = int(ts) for d in dom_list: target = xstransact.Read('/local/domain/' + d + '/target') if int(d) is not ti and target is None : pci_str_list = pci_str_list + get_assigned_pci_devices(int(d)) return pci_str_list class PciController(DevController): def __init__(self, vm): self.aerStateWatch = None DevController.__init__(self, vm) def getDeviceDetails(self, config): """@see DevController.getDeviceDetails""" back = {} pcidevid = 0 pci_defopts = [] if 'pci_msitranslate' in self.vm.info['platform']: pci_defopts.append(['msitranslate', str(self.vm.info['platform']['pci_msitranslate'])]) if 'pci_power_mgmt' in self.vm.info['platform']: pci_defopts.append(['power_mgmt', str(self.vm.info['platform']['pci_power_mgmt'])]) for pci_config in config.get('devs', []): domain = parse_hex(pci_config.get('domain', 0)) bus = parse_hex(pci_config.get('bus', 0)) slot = parse_hex(pci_config.get('slot', 0)) func = parse_hex(pci_config.get('func', 0)) vdevfn = parse_hex(pci_config.get('vdevfn', \ '0x%02x' % AUTO_PHP_SLOT)) optslist = [] if pci_config.has_key('opts'): optslist += pci_config['opts'] if optslist or pci_defopts: opts = serialise_pci_opts( append_default_pci_opts(optslist, pci_defopts)) back['opts-%i' % pcidevid] = opts back['dev-%i' % pcidevid] = "%04x:%02x:%02x.%01x" % \ (domain, bus, slot, func) back['uuid-%i' % pcidevid] = pci_config.get('uuid', '') back['key-%i' % pcidevid] = pci_config.get('key', '') back['vdevfn-%i' % pcidevid] = "%02x" % vdevfn pcidevid += 1 back['num_devs']=str(pcidevid) back['uuid'] = config.get('uuid','') return (0, back, {}) def reconfigureDevice_find(self, devid, nsearch_dev, match_dev): for j in range(nsearch_dev): if match_dev == self.readBackend(devid, 'dev-%i' % j): return j return None def reconfigureDevice(self, _, config): """@see DevController.reconfigureDevice""" (devid, back, front) = self.getDeviceDetails(config) num_devs = int(back['num_devs']) states = config.get('states', []) num_olddevs = int(self.readBackend(devid, 'num_devs')) for i in range(num_devs): try: dev = back['dev-%i' % i] state = states[i] uuid = back['uuid-%i' %i] key = back['key-%i' %i] opts = '' if 'opts-%i' % i in back: opts = back['opts-%i' % i] except: raise XendError('Error reading config') if state == 'Initialising': devno = self.reconfigureDevice_find(devid, num_olddevs, dev) if devno == None: devno = num_olddevs + i log.debug('Attaching PCI device %s.' % dev) attaching = True else: log.debug('Reconfiguring PCI device %s.' % dev) attaching = False self.setupOneDevice(parse_pci_name(dev)) self.writeBackend(devid, 'dev-%i' % devno, dev) self.writeBackend(devid, 'state-%i' % devno, str(xenbusState['Initialising'])) self.writeBackend(devid, 'uuid-%i' % devno, uuid) self.writeBackend(devid, 'key-%i' % devno, key) if len(opts) > 0: self.writeBackend(devid, 'opts-%i' % devno, opts) if back.has_key('vdevfn-%i' % i): self.writeBackend(devid, 'vdevfn-%i' % devno, back['vdevfn-%i' % i]) # If a device is being attached then num_devs will grow if attaching: self.writeBackend(devid, 'num_devs', str(devno + 1)) elif state == 'Closing': # PCI device detachment devno = self.reconfigureDevice_find(devid, num_olddevs, dev) if devno == None: raise XendError('Device %s is not connected' % dev) log.debug('Detaching device %s' % dev) self.writeBackend(devid, 'state-%i' % devno, str(xenbusState['Closing'])) else: raise XendError('Error configuring device %s: invalid state %s' % (dev,state)) self.writeBackend(devid, 'state', str(xenbusState['Reconfiguring'])) return self.readBackend(devid, 'uuid') def getDeviceConfiguration(self, devid, transaction = None): result = DevController.getDeviceConfiguration(self, devid, transaction) num_devs = self.readBackend(devid, 'num_devs') pci_devs = [] for i in range(int(num_devs)): pci_dev = parse_pci_name(self.readBackend(devid, 'dev-%d' % i)) # Per device uuid info pci_dev['uuid'] = self.readBackend(devid, 'uuid-%d' % i) pci_dev['key'] = self.readBackend(devid, 'key-%d' % i) pci_dev['vdevfn'] = '0x%s' % self.readBackend(devid, 'vdevfn-%d' % i) #append opts info opts = self.readBackend(devid, 'opts-%d' % i) if opts is not None: pci_dev['opts'] = opts pci_devs.append(pci_dev) result['devs'] = pci_devs result['uuid'] = self.readBackend(devid, 'uuid') return result def configuration(self, devid, transaction = None): """Returns SXPR for devices on domain. @note: we treat this dict especially to convert to SXP because it is not a straight dict of strings.""" configDict = self.getDeviceConfiguration(devid, transaction) sxpr = [self.deviceClass] # remove devs devs = configDict.pop('devs', []) for dev in devs: dev_sxpr = ['dev'] for dev_key, dev_val in dev.items(): if dev_key == 'opts': opts_sxpr = pci_opts_list_to_sxp(split_pci_opts(dev_val)) dev_sxpr = sxp.merge(dev_sxpr, opts_sxpr) else: dev_sxpr.append([dev_key, dev_val]) sxpr.append(dev_sxpr) for key, val in configDict.items(): if type(val) == type(list()): for v in val: sxpr.append([key, v]) else: sxpr.append([key, val]) return sxpr def CheckSiblingDevices(self, domid, dev): """ Check if all sibling devices of dev are owned by pciback or pci-stub """ if not self.vm.info.is_hvm(): return group_str = xc.get_device_group(domid, dev.domain, dev.bus, dev.slot, dev.func) if group_str == "": return #group string format xx:xx.x,xx:xx.x, for i in group_str.split(','): if i == '': continue pci_dev = parse_pci_name(i) pci_dev['domain'] = '%04x' % dev.domain try: sdev = PciDevice(pci_dev) except Exception, e: #no dom0 drivers bound to sdev continue if sdev.driver!='pciback' and sdev.driver!='pci-stub': raise VmError(("pci: PCI Backend and pci-stub don't "+ \ "own sibling device %s of device %s"\ )%(sdev.name, dev.name)) return def setupOneDevice(self, pci_dev): """ Attach I/O resources for device to frontend domain """ fe_domid = self.getDomid() try: dev = PciDevice(pci_dev) except Exception, e: raise VmError("pci: failed to locate device and "+ "parse its resources - "+str(e)) if dev.driver!='pciback' and dev.driver!='pci-stub': raise VmError(("pci: PCI Backend and pci-stub don't own "+ \ "device %s") %(dev.name)) self.CheckSiblingDevices(fe_domid, dev) if dev.driver == 'pciback': PCIQuirk(dev) if not self.vm.info.is_hvm() and not self.vm.info.is_stubdom() : # Setup IOMMU device assignment bdf = xc.assign_device(fe_domid, pci_dict_to_xc_str(pci_dev)) pci_str = pci_dict_to_bdf_str(pci_dev) if bdf > 0: raise VmError("Failed to assign device to IOMMU (%s)" % pci_str) log.debug("pci: assign device %s" % pci_str) for (start, size) in dev.ioports: log.debug('pci: enabling ioport 0x%x/0x%x'%(start,size)) rc = xc.domain_ioport_permission(domid = fe_domid, first_port = start, nr_ports = size, allow_access = True) if rc<0: raise VmError(('pci: failed to configure I/O ports on device '+ '%s - errno=%d')%(dev.name,rc)) for (start, size) in dev.iomem: # Convert start/size from bytes to page frame sizes start_pfn = start>>PAGE_SHIFT # Round number of pages up to nearest page boundary (if not on one) nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT log.debug('pci: enabling iomem 0x%x/0x%x pfn 0x%x/0x%x'% \ (start,size,start_pfn,nr_pfns)) rc = xc.domain_iomem_permission(domid = fe_domid, first_pfn = start_pfn, nr_pfns = nr_pfns, allow_access = True) if rc<0: raise VmError(('pci: failed to configure I/O memory on device '+ '%s - errno=%d')%(dev.name,rc)) if not self.vm.info.is_hvm() and dev.irq: rc = xc.physdev_map_pirq(domid = fe_domid, index = dev.irq, pirq = dev.irq) if rc < 0: raise VmError(('pci: failed to map irq on device '+ '%s - errno=%d')%(dev.name,rc)) if dev.irq>0: log.debug('pci: enabling irq %d'%dev.irq) rc = xc.domain_irq_permission(domid = fe_domid, pirq = dev.irq, allow_access = True) if rc<0: raise VmError(('pci: failed to configure irq on device '+ '%s - errno=%d')%(dev.name,rc)) def dev_check_assignability_and_do_FLR(self, config): pci_dev_list = config.get('devs', []) pci_str_list = map(pci_dict_to_bdf_str, pci_dev_list) if len(pci_str_list) != len(set(pci_str_list)): raise VmError('pci: duplicate devices specified in guest config?') strict_check = xoptions.get_pci_dev_assign_strict_check() devs = [] for pci_dev in pci_dev_list: try: dev = PciDevice(pci_dev) except Exception, e: raise VmError("pci: failed to locate device and "+ "parse its resources - "+str(e)) if dev.driver!='pciback' and dev.driver!='pci-stub': raise VmError(("pci: PCI Backend and pci-stub don't own device"\ " %s") %(dev.name)) devs.append(dev) if dev.has_non_page_aligned_bar and strict_check: raise VmError("pci: %s: non-page-aligned MMIO BAR found." % dev.name) # Check if there is intermediate PCIe switch bewteen the device and # Root Complex. if self.vm.info.is_hvm() and dev.is_behind_switch_lacking_acs() \ and strict_check: err_msg = 'pci: to avoid potential security issue, %s is not'+\ ' allowed to be assigned to guest since it is behind'+\ ' PCIe switch that does not support or enable ACS.' raise VmError(err_msg % dev.name) if (dev.dev_type == DEV_TYPE_PCIe_ENDPOINT) and not dev.pcie_flr: if dev.bus == 0: # We cope with this case by using the Dstate transition # method or some vendor specific methods for now. err_msg = 'pci: %s: it is on bus 0, but has no PCIe' +\ ' FLR Capability. Will try the Dstate transition'+\ ' method or some vendor specific methods if available.' log.warn(err_msg % dev.name) else: if not self.vm.info.is_hvm(): continue if not strict_check: continue funcs = dev.find_all_the_multi_functions() dev.devs_check_driver(funcs) for f in funcs: if not f in pci_str_list: # f has been assigned to other guest? if f in get_all_assigned_pci_devices(): err_msg = 'pci: %s must be co-assigned to' + \ ' the same guest with %s' raise VmError(err_msg % (f, dev.name)) elif dev.dev_type == DEV_TYPE_PCI: if dev.bus == 0 or arch.type == "ia64": if not dev.pci_af_flr: # We cope with this case by using the Dstate transition # method or some vendor specific methods for now. err_msg = 'pci: %s: it is on bus 0, but has no PCI' +\ ' Advanced Capabilities for FLR. Will try the'+\ ' Dstate transition method or some vendor' +\ ' specific methods if available.' log.warn(err_msg % dev.name) else: if not self.vm.info.is_hvm(): continue if not strict_check: continue # All devices behind the uppermost PCI/PCI-X bridge must be\ # co-assigned to the same guest. devs_str = dev.find_coassigned_pci_devices(True) # Remove the element 0 which is a bridge del devs_str[0] dev.devs_check_driver(devs_str) for s in devs_str: if not s in pci_str_list: # s has been assigned to other guest? if s in get_all_assigned_pci_devices(): err_msg = 'pci: %s must be co-assigned to the'+\ ' same guest with %s' raise VmError(err_msg % (s, dev.name)) # try to do FLR for dev in devs: dev.do_FLR(self.vm.info.is_hvm(), strict_check) def setupDevice(self, config): """Setup devices from config """ pci_dev_list = config.get('devs', []) for d in pci_dev_list: self.setupOneDevice(d) wPath = '/local/domain/0/backend/pci/%u/0/aerState' % (self.getDomid()) self.aerStateWatch = xswatch(wPath, self._handleAerStateWatch) log.debug('pci: register aer watch %s', wPath) return def _handleAerStateWatch(self, _): log.debug('XendDomainInfo.handleAerStateWatch') if self.getDomid() == 0: raise XendError('Domain 0 cannot be shutdown') readPath = '/local/domain/0/backend/pci/%u/0/aerState' % (self.getDomid()) action = xstransact.Read(readPath) if action and action=='aerfail': log.debug('shutdown domain because of aer handle error') self.vm.shutdown('poweroff') return True def cleanupOneDevice(self, pci_dev): """ Detach I/O resources for device from frontend domain """ fe_domid = self.getDomid() try: dev = PciDevice(pci_dev) except Exception, e: raise VmError("pci: failed to locate device and "+ "parse its resources - "+str(e)) if dev.driver!='pciback' and dev.driver!='pci-stub': raise VmError(("pci: PCI Backend and pci-stub don't own device "+ \ "%s") %(dev.name)) # Need to do FLR here before deassign device in order to terminate # DMA transaction, etc dev.do_FLR(self.vm.info.is_hvm(), xoptions.get_pci_dev_assign_strict_check()) if not self.vm.info.is_stubdom() : bdf = xc.deassign_device(fe_domid, pci_dict_to_xc_str(pci_dev)) pci_str = pci_dict_to_bdf_str(pci_dev) if bdf > 0: raise VmError("Failed to deassign device from IOMMU (%s)" % pci_str) log.debug("pci: Deassign device %s" % pci_str) for (start, size) in dev.ioports: log.debug('pci: disabling ioport 0x%x/0x%x'%(start,size)) rc = xc.domain_ioport_permission(domid = fe_domid, first_port = start, nr_ports = size, allow_access = False) if rc<0: raise VmError(('pci: failed to configure I/O ports on device '+ '%s - errno=%d')%(dev.name,rc)) for (start, size) in dev.iomem: # Convert start/size from bytes to page frame sizes start_pfn = start>>PAGE_SHIFT # Round number of pages up to nearest page boundary (if not on one) nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT log.debug('pci: disabling iomem 0x%x/0x%x pfn 0x%x/0x%x'% \ (start,size,start_pfn,nr_pfns)) rc = xc.domain_iomem_permission(domid = fe_domid, first_pfn = start_pfn, nr_pfns = nr_pfns, allow_access = False) if rc<0: raise VmError(('pci: failed to configure I/O memory on device '+ '%s - errno=%d')%(dev.name,rc)) if dev.irq>0: log.debug('pci: disabling irq %d'%dev.irq) rc = xc.domain_irq_permission(domid = fe_domid, pirq = dev.irq, allow_access = False) if rc<0: raise VmError(('pci: failed to configure irq on device '+ '%s - errno=%d')%(dev.name,rc)) def cleanupDevice(self, devid): """ Detach I/O resources for device and cleanup xenstore nodes after reconfigure. @param devid: The device ID @type devid: int @return: Return the number of devices connected @rtype: int """ num_devs = int(self.readBackend(devid, 'num_devs')) new_num_devs = 0 for i in range(num_devs): try: state = int(self.readBackend(devid, 'state-%i' % i)) except: state = xenbusState['Unknown'] if state == xenbusState['Closing']: # Detach I/O resources. pci_dev = parse_pci_name(self.readBackend(devid, 'dev-%i' % i)) # In HVM case, I/O resources are disabled in ioemu. self.cleanupOneDevice(pci_dev) # Remove xenstore nodes. list = ['dev', 'vdev', 'state', 'uuid', 'vdevfn', 'key'] if self.readBackend(devid, 'opts-%i' % i) is not None: list.append('opts') for key in list: self.removeBackend(devid, '%s-%i' % (key, i)) else: new_num_devs = new_num_devs + 1 if new_num_devs == i + 1: continue list = ['dev', 'vdev', 'state', 'uuid', 'opts', 'vdevfn', 'key'] for key in list: tmp = self.readBackend(devid, '%s-%i' % (key, i)) if tmp is None: continue self.removeBackend(devid, '%s-%i' % (key, i)) self.writeBackend(devid, '%s-%i' % (key, new_num_devs - 1), tmp) self.writeBackend(devid, 'num_devs', str(new_num_devs)) return new_num_devs def destroyDevice(self, devid, force): DevController.destroyDevice(self, devid, True) log.debug('pci: unregister aer watch') self.unwatchAerState() def unwatchAerState(self): """Remove the watch on the domain's aerState node, if any.""" try: try: if self.aerStateWatch: self.aerStateWatch.unwatch() finally: self.aerStateWatch = None except: log.exception("Unwatching aerState failed.") def waitForBackend(self,devid): return (0, "ok - no hotplug") def migrate(self, config, network, dst, step, domName): raise XendError('Migration not permitted with assigned PCI device.')