PCIe 学习笔记(四)

pci note 4
==========

-v0.1 2015.6.20 Sherlock draft: about pci resource assignment, just analysis
                                ARM specific PCIe
-v0.2 2015.6.22 Sherlock add introduction, some details analysis

有些机器没有中文输入法，就成了现在中英杂揉了 :(
linux kernel中pci分配资源的代码比较复杂，下面先总体介绍大体的流程，有了一个整体
概念后再去理解细节就比较容易一些了。

整个资源分配的过程就是从系统的总资源里给每个pci设备的bar分配资源，给每个pci桥
的base, limit的寄存器分配资源。借用这个系列笔记一中的图，资源分配所要做的就是填写
pci ep设备和pci桥中配置空间的寄存器，从总的资源(这里假设soc系统给这个pci host
bridge分配的资源是 0xb200_0000 ~ 0xb400_0000)给pci桥上的base/limit寄存器分资源，
给pci ep设备上的Bar分资源。

现在已知的是：
1. resource: 0xb200_0000 ~ 0xb400_0000. 这个是系统早就分好的。pcie dts节点中的range项是其的一个子集。

2. pci ep设备有多少个bar和各个bar的信息, 这些信息已经存在了对应的pci_dev->resource[]中了。

		    +----------------+ ----> PCIe host bridge| pcie root port |+----------------+ ----> in Soc|           (resource: 0xb200_0000 ~ 0xb400_0000)+-------------------------------------------------+ ----> switch|		    +----------------+                ||		    |   pci bridge   |                ||		    +----------------+                ||			    |                         ||         -------------------------------         ||         |                             |         || +----------------+	   +----------------+ |  io base/limit| |   pci bridge   |	   |   pci bridge   | |  32bits mem base/limit| +----------------+	   +----------------+ |  64bits mem base/limit+-------------------------------------------------+  pref 64bits mem base/limit|                             |+----------------+           +----------------+|  PCIe net card |           | PCIe net card  |+----------------+           +----------------+|     BAR 0      |           |     BAR 0      |+----------------+           +----------------+|     ...        |           |     ...        |+----------------+           +----------------+|     BAR 5      |           |     BAR 5      |+----------------+           +----------------+

大体流程是：
所有分配可以用pci_assign_unassigned_bus_resources完成。
__pci_bus_size_bridges用深度优先递归确定各级pci桥上base/limit的大小。会记录在
pci_dev->resource[PCI_BRIDGE_RESOURCES] ...中，这时并没有在寄存器中写入数值。
__pci_bus_assign_resources()首先对当前总线下的设备请求的资源排序，这个资源中包括
总线下的设备上的bar; 总线下游请求的资源,即base/limit下的资源。对于该总线下设备
上的bar资源，在下面__assign_resources_sorted的调用链中立即分配, pci桥上的base/limit
则先不分配。__pci_bus_assign_resources()中再次用深度优先递归的办法，依次分配各个
pci ep设备上的bar资源，在每个递归向上返回的过程中调用__pci_bus_size_bridges()
设置pci桥上的base/limit寄存器。

下面用具体例子再详细解释一下分配过程, 是用的例子是直接在pci host bridge上插
一个pcie ep设备：

		|   root bus: 0   ---->  struct pci_bus|+----------------+        ---->  struct pci_host_bridge| pcie root port |        ---->  struct pci_dev+----------------+        |   bus: 1        ---->  struct pci_bus|+----------------+| pcie net cards |        ---->  struct pci_dev+----------------+

void pci_assign_unassigned_bus_resources(struct pci_bus *bus)
{struct pci_dev *dev;LIST_HEAD(add_list); /* list of resources thatwant additional resources */down_read(&pci_bus_sem);/* bus is root pci bus, so bus->devices is pci host bridge's pci_dev */list_for_each_entry(dev, &bus->devices, bus_list)if (pci_is_bridge(dev) && pci_has_subordinate(dev))/* dev->subordinate is pci_bus: 1, pcie device* is connected to pci bus 1(assuming root bus* is 0)*/__pci_bus_size_bridges(dev->subordinate,&add_list);up_read(&pci_bus_sem);/* bus 0 */__pci_bus_assign_resources(bus, &add_list, NULL);--> pbus_assign_resources_sorted(bus, realloc_head, fail_head);/* will parse the size of resource(each bar), and add* them to list head using struct pci_dev_resource.* * bus is root bus, so dev is pci host bridge,* but comments in __dev_sort_resources said that* we do not parse the resources in host bridge* * in pcie-designware, it does not use PCI_CLASS_BRIDGE_HOST* to indicate PCIe host bridge, but use PCI_CLASS_BRIDGE_PCI.* so here will parse "Bar" resources in host bridge, and* resaults will be stored in pci_dev->resource[PCI_BRIDGE_RESOURCES]...* so pci_assign_resource will assign Bar 7 and Bar 8 according* to pci_dev->resource[PCI_BRIDGE_RESOURCES]..., in fact, this* info should be a base/limit info. And in pci_assign_resource* it will not allow above base/limit data to be wrotten to bar* * So we need to find a way to fix above problem*/--> list_for_each_entry(dev, &bus->devices, bus_list)__dev_sort_resources(dev, &head);--> __assign_resources_sorted(&head, realloc_head, fail_head);--> ...--> assign_requested_resources_sorted(head, fail_head);--> list_for_each_entry(dev_res, head, list)--> pci_assign_resource(dev_res->dev, idx)--> list_for_each_entry(dev, &bus->devices, bus_list)/* 这里会递归进入下一级总线，对已有的bar资源排序，然后分配。* pci_assign_resource会对bar写入相应的资源及地址*/--> __pci_bus_assign_resources(b, realloc_head, fail_head);.../* 从__pci_bus_assign_resources返回进入pci_setup_bridge分配pci* bridge的base/limit资源*/--> case PCI_CLASS_BRIDGE_PCI:--> pci_setup_bridge(b);BUG_ON(!list_empty(&add_list));
}/* I am afraid that below function added pci_dev->resource[PCI_BRIDGE_RESOURCES]** dev->subordinate: pci_bus 1*/
--> __pci_bus_size_bridges(dev->subordinate, &add_list);/* bus: pci_bus 1, here we find pci_bus 1 and go out this list_for_each_entry */--> list_for_each_entry(dev, &bus->devices, bus_list)--> struct pci_bus *b = dev->subordinate;if (!b)continue;...--> case PCI_CLASS_BRIDGE_PCI:default:__pci_bus_size_bridges(b, realloc_head);--> switch (bus->self->class >> 8).../* check if this bridge support io and prefetch mem ranges *//**  log if an intel 82575 plugged in above pci host bridge:**  begin: pci_bridge_check_ranges*  in pci_bridge_check_ranges: mem size: 1*  in pci_bridge_check_ranges: first read io: 0*  in pci_bridge_check_ranges: second read io: e0f0*  in pci_bridge_check_ranges: io size: 1*  in pci_bridge_check_ranges: first read pref mem: 10001*  in pci_bridge_check_ranges: first read 64 pref mem: 0*  in pci_bridge_check_ranges: second read 64 pref mem: ffffffff*/--> case PCI_CLASS_BRIDGE_PCIpci_bridge_check_ranges(bus);/* why writing 0xe0f0, so here just write a* random number to test if this register can* be wrotten something?*/--> pci_write_config_word(bridge, PCI_IO_BASE, 0xe0f0);--> default:/* to caculate size of io and mem ranges of this pci bridge */--> pbus_size_io()/* pci_dev->resource[0] is 32 mem ? */--> b_res = &bus->self->resource[PCI_BRIDGE_RESOURCES];mask = IORESOURCE_MEM;prefmask = IORESOURCE_MEM | IORESOURCE_PREFETCH;/* pci_dev->resource[2] is 64 mem ? */--> if (b_res[2].flags & IORESOURCE_MEM_64)...pbus_size_mem()--> pbus_size_io(struct pci_bus *bus, resource_size_t min_size,resource_size_t add_size, struct list_head *realloc_head)/* below function will return pci_bus->resource[], here will be io* resource item*/--> find_free_bus_resource()/* from comments in kernel code: I/O windows are 4K-aligned, but some* bridges have an extension to support 1K alignment. ** min_align is 4k here.*/--> min_align = window_alignment(bus, IORESOURCE_IO);--> list_for_each_entry(dev, &bus->devices, bus_list)/* find required io resource in pci_dev->resource */--> for (i = 0; i < PCI_NUM_RESOURCES; i++).../* add io resource to pci_bus->resource */--> b_res->start = min_align;b_res->end = b_res->start + size0 - 1;b_res->flags |= IORESOURCE_STARTALIGN;--> assign_requested_resources_sorted(head, fail_head);/* For resources in host bridge, head here has bar0, bar7 and bar8.* I do not know when bar7 and bar8 resources had been added to head** I added some prints, it seems that before resource assignment* (__pci_bus_assign_resources) bar7 and bar8 already added to* pci_dev->resource[PCI_BRIDGE_RESOURCES] and [PCI_BRIDGE_RESOURCES + 1]*/--> list_for_each_entry(dev_res, head, list)--> pci_assign_resource(dev_res->dev, idx)