lk/linux-kernel-test
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branches 'core/iommu', 'x86/amd-iommu' and 'x86/iommu' into x86-v28-for-linus-phase3-B

Browse Source 
Conflicts:
	arch/x86/kernel/pci-gart_64.c
	include/asm-x86/dma-mapping.h
This commit is contained in:
Ingo Molnar
2008-10-10 19:47:12 +02:00
parent 3dd392a407 72d31053f6 129d6aba44 1e19b16a30
commit 725c25819e
25 changed files with 803 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files

328
arch/x86/kernel/amd_iommu.c
View File

@ -33,6 +33,10 @@
static DEFINE_RWLOCK(amd_iommu_devtable_lock);
/* A list of preallocated protection domains */
static LIST_HEAD(iommu_pd_list);
static DEFINE_SPINLOCK(iommu_pd_list_lock);
/*
* general struct to manage commands send to an IOMMU
*/
@ -49,6 +53,102 @@ static int iommu_has_npcache(struct amd_iommu *iommu)
return iommu->cap & IOMMU_CAP_NPCACHE;
}
/****************************************************************************
*
* Interrupt handling functions
*
****************************************************************************/
static void iommu_print_event(void *__evt)
{
u32 *event = __evt;
int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
u64 address = (u64)(((u64)event[3]) << 32) | event[2];
printk(KERN_ERR "AMD IOMMU: Event logged [");
switch (type) {
case EVENT_TYPE_ILL_DEV:
printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
"address=0x%016llx flags=0x%04x]\n",
PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
address, flags);
break;
case EVENT_TYPE_IO_FAULT:
printk("IO_PAGE_FAULT device=%02x:%02x.%x "
"domain=0x%04x address=0x%016llx flags=0x%04x]\n",
PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
domid, address, flags);
break;
case EVENT_TYPE_DEV_TAB_ERR:
printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
"address=0x%016llx flags=0x%04x]\n",
PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
address, flags);
break;
case EVENT_TYPE_PAGE_TAB_ERR:
printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
"domain=0x%04x address=0x%016llx flags=0x%04x]\n",
PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
domid, address, flags);
break;
case EVENT_TYPE_ILL_CMD:
printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
break;
case EVENT_TYPE_CMD_HARD_ERR:
printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
"flags=0x%04x]\n", address, flags);
break;
case EVENT_TYPE_IOTLB_INV_TO:
printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
"address=0x%016llx]\n",
PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
address);
break;
case EVENT_TYPE_INV_DEV_REQ:
printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
"address=0x%016llx flags=0x%04x]\n",
PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
address, flags);
break;
default:
printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
}
}
static void iommu_poll_events(struct amd_iommu *iommu)
{
u32 head, tail;
unsigned long flags;
spin_lock_irqsave(&iommu->lock, flags);
head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
while (head != tail) {
iommu_print_event(iommu->evt_buf + head);
head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
}
writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
spin_unlock_irqrestore(&iommu->lock, flags);
}
irqreturn_t amd_iommu_int_handler(int irq, void *data)
{
struct amd_iommu *iommu;
list_for_each_entry(iommu, &amd_iommu_list, list)
iommu_poll_events(iommu);
return IRQ_HANDLED;
}
/****************************************************************************
*
* IOMMU command queuing functions
@ -213,6 +313,14 @@ static int iommu_flush_pages(struct amd_iommu *iommu, u16 domid,
return 0;
}
/* Flush the whole IO/TLB for a given protection domain */
static void iommu_flush_tlb(struct amd_iommu *iommu, u16 domid)
{
u64 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
iommu_queue_inv_iommu_pages(iommu, address, domid, 0, 1);
}
/****************************************************************************
*
* The functions below are used the create the page table mappings for
@ -372,11 +480,6 @@ static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
* efficient allocator.
*
****************************************************************************/
static unsigned long dma_mask_to_pages(unsigned long mask)
{
return (mask >> PAGE_SHIFT) +
(PAGE_ALIGN(mask & ~PAGE_MASK) >> PAGE_SHIFT);
}
/*
* The address allocator core function.
@ -385,25 +488,31 @@ static unsigned long dma_mask_to_pages(unsigned long mask)
*/
static unsigned long dma_ops_alloc_addresses(struct device *dev,
struct dma_ops_domain *dom,
unsigned int pages)
unsigned int pages,
unsigned long align_mask,
u64 dma_mask)
{
unsigned long limit = dma_mask_to_pages(*dev->dma_mask);
unsigned long limit;
unsigned long address;
unsigned long size = dom->aperture_size >> PAGE_SHIFT;
unsigned long boundary_size;
boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
PAGE_SIZE) >> PAGE_SHIFT;
limit = limit < size ? limit : size;
limit = iommu_device_max_index(dom->aperture_size >> PAGE_SHIFT, 0,
dma_mask >> PAGE_SHIFT);
if (dom->next_bit >= limit)
if (dom->next_bit >= limit) {
dom->next_bit = 0;
dom->need_flush = true;
}
address = iommu_area_alloc(dom->bitmap, limit, dom->next_bit, pages,
0 , boundary_size, 0);
if (address == -1)
0 , boundary_size, align_mask);
if (address == -1) {
address = iommu_area_alloc(dom->bitmap, limit, 0, pages,
0, boundary_size, 0);
0, boundary_size, align_mask);
dom->need_flush = true;
}
if (likely(address != -1)) {
dom->next_bit = address + pages;
@ -469,7 +578,7 @@ static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
if (start_page + pages > last_page)
pages = last_page - start_page;
set_bit_string(dom->bitmap, start_page, pages);
iommu_area_reserve(dom->bitmap, start_page, pages);
}
static void dma_ops_free_pagetable(struct dma_ops_domain *dma_dom)
@ -563,6 +672,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(struct amd_iommu *iommu,
dma_dom->bitmap[0] = 1;
dma_dom->next_bit = 0;
dma_dom->need_flush = false;
dma_dom->target_dev = 0xffff;
/* Intialize the exclusion range if necessary */
if (iommu->exclusion_start &&
iommu->exclusion_start < dma_dom->aperture_size) {
@ -633,12 +745,13 @@ static void set_device_domain(struct amd_iommu *iommu,
u64 pte_root = virt_to_phys(domain->pt_root);
pte_root |= (domain->mode & 0x07) << 9;
pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | 2;
pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
<< DEV_ENTRY_MODE_SHIFT;
pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
write_lock_irqsave(&amd_iommu_devtable_lock, flags);
amd_iommu_dev_table[devid].data[0] = pte_root;
amd_iommu_dev_table[devid].data[1] = pte_root >> 32;
amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root);
amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root);
amd_iommu_dev_table[devid].data[2] = domain->id;
amd_iommu_pd_table[devid] = domain;
@ -655,6 +768,45 @@ static void set_device_domain(struct amd_iommu *iommu,
*
*****************************************************************************/
/*
* This function checks if the driver got a valid device from the caller to
* avoid dereferencing invalid pointers.
*/
static bool check_device(struct device *dev)
{
if (!dev || !dev->dma_mask)
return false;
return true;
}
/*
* In this function the list of preallocated protection domains is traversed to
* find the domain for a specific device
*/
static struct dma_ops_domain *find_protection_domain(u16 devid)
{
struct dma_ops_domain *entry, *ret = NULL;
unsigned long flags;
if (list_empty(&iommu_pd_list))
return NULL;
spin_lock_irqsave(&iommu_pd_list_lock, flags);
list_for_each_entry(entry, &iommu_pd_list, list) {
if (entry->target_dev == devid) {
ret = entry;
list_del(&ret->list);
break;
}
}
spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
return ret;
}
/*
* In the dma_ops path we only have the struct device. This function
* finds the corresponding IOMMU, the protection domain and the
@ -671,27 +823,30 @@ static int get_device_resources(struct device *dev,
struct pci_dev *pcidev;
u16 _bdf;
BUG_ON(!dev || dev->bus != &pci_bus_type || !dev->dma_mask);
*iommu = NULL;
*domain = NULL;
*bdf = 0xffff;
if (dev->bus != &pci_bus_type)
return 0;
pcidev = to_pci_dev(dev);
_bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
/* device not translated by any IOMMU in the system? */
if (_bdf > amd_iommu_last_bdf) {
*iommu = NULL;
*domain = NULL;
*bdf = 0xffff;
if (_bdf > amd_iommu_last_bdf)
return 0;
}
*bdf = amd_iommu_alias_table[_bdf];
*iommu = amd_iommu_rlookup_table[*bdf];
if (*iommu == NULL)
return 0;
dma_dom = (*iommu)->default_dom;
*domain = domain_for_device(*bdf);
if (*domain == NULL) {
dma_dom = find_protection_domain(*bdf);
if (!dma_dom)
dma_dom = (*iommu)->default_dom;
*domain = &dma_dom->domain;
set_device_domain(*iommu, *domain, *bdf);
printk(KERN_INFO "AMD IOMMU: Using protection domain %d for "
@ -770,17 +925,24 @@ static dma_addr_t __map_single(struct device *dev,
struct dma_ops_domain *dma_dom,
phys_addr_t paddr,
size_t size,
int dir)
int dir,
bool align,
u64 dma_mask)
{
dma_addr_t offset = paddr & ~PAGE_MASK;
dma_addr_t address, start;
unsigned int pages;
unsigned long align_mask = 0;
int i;
pages = iommu_num_pages(paddr, size);
paddr &= PAGE_MASK;
address = dma_ops_alloc_addresses(dev, dma_dom, pages);
if (align)
align_mask = (1UL << get_order(size)) - 1;
address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
dma_mask);
if (unlikely(address == bad_dma_address))
goto out;
@ -792,6 +954,12 @@ static dma_addr_t __map_single(struct device *dev,
}
address += offset;
if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
iommu_flush_tlb(iommu, dma_dom->domain.id);
dma_dom->need_flush = false;
} else if (unlikely(iommu_has_npcache(iommu)))
iommu_flush_pages(iommu, dma_dom->domain.id, address, size);
out:
return address;
}
@ -822,6 +990,9 @@ static void __unmap_single(struct amd_iommu *iommu,
}
dma_ops_free_addresses(dma_dom, dma_addr, pages);
if (amd_iommu_unmap_flush)
iommu_flush_pages(iommu, dma_dom->domain.id, dma_addr, size);
}
/*
@ -835,6 +1006,12 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
struct protection_domain *domain;
u16 devid;
dma_addr_t addr;
u64 dma_mask;
if (!check_device(dev))
return bad_dma_address;
dma_mask = *dev->dma_mask;
get_device_resources(dev, &iommu, &domain, &devid);
@ -843,14 +1020,12 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr,
return (dma_addr_t)paddr;
spin_lock_irqsave(&domain->lock, flags);
addr = __map_single(dev, iommu, domain->priv, paddr, size, dir);
addr = __map_single(dev, iommu, domain->priv, paddr, size, dir, false,
dma_mask);
if (addr == bad_dma_address)
goto out;
if (iommu_has_npcache(iommu))
iommu_flush_pages(iommu, domain->id, addr, size);
if (iommu->need_sync)
if (unlikely(iommu->need_sync))
iommu_completion_wait(iommu);
out:
@ -870,7 +1045,8 @@ static void unmap_single(struct device *dev, dma_addr_t dma_addr,
struct protection_domain *domain;
u16 devid;
if (!get_device_resources(dev, &iommu, &domain, &devid))
if (!check_device(dev) ||
!get_device_resources(dev, &iommu, &domain, &devid))
/* device not handled by any AMD IOMMU */
return;
@ -878,9 +1054,7 @@ static void unmap_single(struct device *dev, dma_addr_t dma_addr,
__unmap_single(iommu, domain->priv, dma_addr, size, dir);
iommu_flush_pages(iommu, domain->id, dma_addr, size);
if (iommu->need_sync)
if (unlikely(iommu->need_sync))
iommu_completion_wait(iommu);
spin_unlock_irqrestore(&domain->lock, flags);
@ -919,6 +1093,12 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
struct scatterlist *s;
phys_addr_t paddr;
int mapped_elems = 0;
u64 dma_mask;
if (!check_device(dev))
return 0;
dma_mask = *dev->dma_mask;
get_device_resources(dev, &iommu, &domain, &devid);
@ -931,19 +1111,17 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
paddr = sg_phys(s);
s->dma_address = __map_single(dev, iommu, domain->priv,
paddr, s->length, dir);
paddr, s->length, dir, false,
dma_mask);
if (s->dma_address) {
s->dma_length = s->length;
mapped_elems++;
} else
goto unmap;
if (iommu_has_npcache(iommu))
iommu_flush_pages(iommu, domain->id, s->dma_address,
s->dma_length);
}
if (iommu->need_sync)
if (unlikely(iommu->need_sync))
iommu_completion_wait(iommu);
out:
@ -977,7 +1155,8 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
u16 devid;
int i;
if (!get_device_resources(dev, &iommu, &domain, &devid))
if (!check_device(dev) ||
!get_device_resources(dev, &iommu, &domain, &devid))
return;
spin_lock_irqsave(&domain->lock, flags);
@ -985,12 +1164,10 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
for_each_sg(sglist, s, nelems, i) {
__unmap_single(iommu, domain->priv, s->dma_address,
s->dma_length, dir);
iommu_flush_pages(iommu, domain->id, s->dma_address,
s->dma_length);
s->dma_address = s->dma_length = 0;
}
if (iommu->need_sync)
if (unlikely(iommu->need_sync))
iommu_completion_wait(iommu);
spin_unlock_irqrestore(&domain->lock, flags);
@ -1008,25 +1185,33 @@ static void *alloc_coherent(struct device *dev, size_t size,
struct protection_domain *domain;
u16 devid;
phys_addr_t paddr;
u64 dma_mask = dev->coherent_dma_mask;
if (!check_device(dev))
return NULL;
if (!get_device_resources(dev, &iommu, &domain, &devid))
flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
flag |= __GFP_ZERO;
virt_addr = (void *)__get_free_pages(flag, get_order(size));
if (!virt_addr)
return 0;
memset(virt_addr, 0, size);
paddr = virt_to_phys(virt_addr);
get_device_resources(dev, &iommu, &domain, &devid);
if (!iommu || !domain) {
*dma_addr = (dma_addr_t)paddr;
return virt_addr;
}
if (!dma_mask)
dma_mask = *dev->dma_mask;
spin_lock_irqsave(&domain->lock, flags);
*dma_addr = __map_single(dev, iommu, domain->priv, paddr,
size, DMA_BIDIRECTIONAL);
size, DMA_BIDIRECTIONAL, true, dma_mask);
if (*dma_addr == bad_dma_address) {
free_pages((unsigned long)virt_addr, get_order(size));
@ -1034,10 +1219,7 @@ static void *alloc_coherent(struct device *dev, size_t size,
goto out;
}
if (iommu_has_npcache(iommu))
iommu_flush_pages(iommu, domain->id, *dma_addr, size);
if (iommu->need_sync)
if (unlikely(iommu->need_sync))
iommu_completion_wait(iommu);
out:
@ -1048,8 +1230,6 @@ out:
/*
* The exported free_coherent function for dma_ops.
* FIXME: fix the generic x86 DMA layer so that it actually calls that
* function.
*/
static void free_coherent(struct device *dev, size_t size,
void *virt_addr, dma_addr_t dma_addr)
@ -1059,6 +1239,9 @@ static void free_coherent(struct device *dev, size_t size,
struct protection_domain *domain;
u16 devid;
if (!check_device(dev))
return;
get_device_resources(dev, &iommu, &domain, &devid);
if (!iommu || !domain)
@ -1067,9 +1250,8 @@ static void free_coherent(struct device *dev, size_t size,
spin_lock_irqsave(&domain->lock, flags);
__unmap_single(iommu, domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
iommu_flush_pages(iommu, domain->id, dma_addr, size);
if (iommu->need_sync)
if (unlikely(iommu->need_sync))
iommu_completion_wait(iommu);
spin_unlock_irqrestore(&domain->lock, flags);
@ -1078,6 +1260,30 @@ free_mem:
free_pages((unsigned long)virt_addr, get_order(size));
}
/*
* This function is called by the DMA layer to find out if we can handle a
* particular device. It is part of the dma_ops.
*/
static int amd_iommu_dma_supported(struct device *dev, u64 mask)
{
u16 bdf;
struct pci_dev *pcidev;
/* No device or no PCI device */
if (!dev || dev->bus != &pci_bus_type)
return 0;
pcidev = to_pci_dev(dev);
bdf = calc_devid(pcidev->bus->number, pcidev->devfn);
/* Out of our scope? */
if (bdf > amd_iommu_last_bdf)
return 0;
return 1;
}
/*
* The function for pre-allocating protection domains.
*
@ -1107,10 +1313,9 @@ void prealloc_protection_domains(void)
if (!dma_dom)
continue;
init_unity_mappings_for_device(dma_dom, devid);
set_device_domain(iommu, &dma_dom->domain, devid);
printk(KERN_INFO "AMD IOMMU: Allocated domain %d for device ",
dma_dom->domain.id);
print_devid(devid, 1);
dma_dom->target_dev = devid;
list_add_tail(&dma_dom->list, &iommu_pd_list);
}
}
@ -1121,6 +1326,7 @@ static struct dma_mapping_ops amd_iommu_dma_ops = {
.unmap_single = unmap_single,
.map_sg = map_sg,
.unmap_sg = unmap_sg,
.dma_supported = amd_iommu_dma_supported,
};
/*