longkgsl_ioctl_gpumem_alloc(structkgsl_device_private*dev_priv,unsignedint cmd,void*data){structkgsl_device*device = dev_priv->device;// ioctl命令参数structkgsl_gpumem_alloc*param = data;// kgsl_mem_entry用于描述用户空间的内存分配[见2.1节]structkgsl_mem_entry*entry;uint64_t flags = param->flags;/*
* On 64 bit kernel, secure memory region is expanded and
* moved to 64 bit address, 32 bit apps can not access it from
* this IOCTL.
*/if((param->flags & KGSL_MEMFLAGS_SECURE)&&is_compat_task()&&test_bit(KGSL_MMU_64BIT,&device->mmu.features))return-EOPNOTSUPP;/* Legacy functions doesn't support these advanced features */
flags &=~((uint64_t) KGSL_MEMFLAGS_USE_CPU_MAP);if(is_compat_task())
flags |= KGSL_MEMFLAGS_FORCE_32BIT;// 创建kgsl_mem_entry[见2.2节]
entry =gpumem_alloc_entry(dev_priv,(uint64_t) param->size, flags);if(IS_ERR(entry))returnPTR_ERR(entry);// 更新参数
param->gpuaddr =(unsignedlong) entry->memdesc.gpuaddr;
param->size =(size_t) entry->memdesc.size;
param->flags =(unsignedint) entry->memdesc.flags;/* Put the extra ref from kgsl_mem_entry_create() */// 减少引用计数, 如果引用计数减为0则通过kgsl_mem_entry_destroy释放kgsl_mem_entrykgsl_mem_entry_put(entry);return0;}
2.1 kgsl_mem_entry
/*
* struct kgsl_mem_entry - a userspace memory allocation
*/structkgsl_mem_entry{// Currently userspace can only hold a single reference count but the kernel may hold morestructkref refcount;// description of the memory[见2.1.1节]structkgsl_memdesc memdesc;// type-specific data, such as the dma-buf attachment pointervoid*priv_data;// rb_node for the gpu address lookup rb treestructrb_node node;// idr index for this entry, can be used to find memory that does not have a valid GPU addressunsignedint id;// 持有该内存的进程structkgsl_process_private*priv;// if !0, userspace requested that his memory be freed, but there are still references to itint pending_free;// String containing user specified metadata for the entrychar metadata[KGSL_GPUOBJ_ALLOC_METADATA_MAX +1];// used to schedule a kgsl_mem_entry_put in atomic contextsstructwork_struct work;/**
* @map_count: Count how many vmas this object is mapped in - used for
* debugfs accounting
*/// 映射的VMA数量atomic_t map_count;};
2.1.1 kgsl_memdesc
/**
* struct kgsl_memdesc - GPU memory object descriptor
*/structkgsl_memdesc{// Pointer to the pagetable that the object is mapped instructkgsl_pagetable*pagetable;// Kernel virtual addressvoid*hostptr;// Number of threads using hostptrunsignedint hostptr_count;// GPU virtual addressuint64_t gpuaddr;// Physical address of the memory objectphys_addr_t physaddr;// Size of the memory objectuint64_t size;// Internal flags and settingsunsignedint priv;structsg_table*sgt;// Function hooks for the memdesc memory type[见2.1.2节]conststructkgsl_memdesc_ops*ops;// Flags set from userspaceuint64_t flags;structdevice*dev;// dma attributes for this memoryunsignedlong attrs;// An array of pointers to allocated pagesstructpage**pages;// Total number of pages allocatedunsignedint page_count;/*
* @lock: Spinlock to protect the gpuaddr from being accessed by
* multiple entities trying to map the same SVM region at once
*/spinlock_t lock;/** @shmem_filp: Pointer to the shmem file backing this memdesc */// 共享内存的文件structfile*shmem_filp;/** @ranges: rbtree base for the interval list of vbo ranges */structrb_root_cached ranges;/** @ranges_lock: Mutex to protect the range database */structmutex ranges_lock;/** @gmuaddr: GMU VA if this is mapped in GMU */
u32 gmuaddr;};
2.1.2 kgsl_memdesc_ops
structkgsl_memdesc_ops{unsignedint vmflags;vm_fault_t(*vmfault)(structkgsl_memdesc*memdesc,structvm_area_struct*vma,structvm_fault*vmf);void(*free)(structkgsl_memdesc*memdesc);int(*map_kernel)(structkgsl_memdesc*memdesc);void(*unmap_kernel)(structkgsl_memdesc*memdesc);/**
* @put_gpuaddr: Put away the GPU address and unmap the memory
* descriptor
*/void(*put_gpuaddr)(structkgsl_memdesc*memdesc);};
2.2 gpumem_alloc_entry
structkgsl_mem_entry*gpumem_alloc_entry(structkgsl_device_private*dev_priv,uint64_t size,uint64_t flags){int ret;structkgsl_process_private*private = dev_priv->process_priv;structkgsl_mem_entry*entry;structkgsl_device*device = dev_priv->device;
u32 cachemode;/* For 32-bit kernel world nothing to do with this flag */if(BITS_PER_LONG ==32)
flags &=~((uint64_t) KGSL_MEMFLAGS_FORCE_32BIT);if(flags & KGSL_MEMFLAGS_VBO)returngpumem_alloc_vbo_entry(dev_priv, size, flags);
flags &= KGSL_MEMFLAGS_GPUREADONLY
| KGSL_CACHEMODE_MASK
| KGSL_MEMTYPE_MASK
| KGSL_MEMALIGN_MASK
| KGSL_MEMFLAGS_USE_CPU_MAP
| KGSL_MEMFLAGS_SECURE
| KGSL_MEMFLAGS_FORCE_32BIT
| KGSL_MEMFLAGS_IOCOHERENT
| KGSL_MEMFLAGS_GUARD_PAGE;/* Return not supported error if secure memory isn't enabled */if((flags & KGSL_MEMFLAGS_SECURE)&&!check_and_warn_secured(device))returnERR_PTR(-EOPNOTSUPP);
flags =cap_alignment(device, flags);/* For now only allow allocations up to 4G */if(size ==0|| size > UINT_MAX)returnERR_PTR(-EINVAL);// 更新缓存策略
flags =kgsl_filter_cachemode(flags);// 前面主要完成标志位的校验和更新// 这里开始创建kgsl_mem_entry[见2.2.1节]
entry =kgsl_mem_entry_create();if(entry ==NULL)returnERR_PTR(-ENOMEM);// 根据标志位判断是否是cached bufferif(IS_ENABLED(CONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT)&&kgsl_cachemode_is_cached(flags))
flags |= KGSL_MEMFLAGS_IOCOHERENT;// 用户空间分配[2.2.2节]
ret =kgsl_allocate_user(device,&entry->memdesc,
size, flags,0);if(ret !=0)goto err;// 绑定映射[2.2.7节]
ret =kgsl_mem_entry_attach_and_map(device, private, entry);if(ret !=0){kgsl_sharedmem_free(&entry->memdesc);goto err;}// 获取缓存模式
cachemode =kgsl_memdesc_get_cachemode(&entry->memdesc);/*
* Secure buffers cannot be reclaimed. Avoid reclaim of cached buffers
* as we could get request for cache operations on these buffers when
* they are reclaimed.
*/// 确认memdesc的pages是否能够直接回收if(!(flags & KGSL_MEMFLAGS_SECURE)&&!(cachemode == KGSL_CACHEMODE_WRITEBACK)&&!(cachemode == KGSL_CACHEMODE_WRITETHROUGH))
entry->memdesc.priv |= KGSL_MEMDESC_CAN_RECLAIM;// 首先确定kgsl_memdesc的buffer类型// 然后将其大小统计进kgsl_process_private的stats数组kgsl_process_add_stats(private,kgsl_memdesc_usermem_type(&entry->memdesc),
entry->memdesc.size);trace_kgsl_mem_alloc(entry);// 将kgsl_mem_entry提交到kgsl_process_private, 以便其他操作也能够访问kgsl_mem_entry_commit_process(entry);return entry;
err:kfree(entry);returnERR_PTR(ret);}
2.2.1 kgsl_mem_entry_create
staticstructkgsl_mem_entry*kgsl_mem_entry_create(void){// 创建kgsl_mem_entrystructkgsl_mem_entry*entry =kzalloc(sizeof(*entry), GFP_KERNEL);if(entry !=NULL){// 初始化kgsl_mem_entry引用计数为1kref_init(&entry->refcount);/* put this ref in userspace memory alloc and map ioctls */// 引用计数加1kref_get(&entry->refcount);// 初始化映射的VMA数量为0atomic_set(&entry->map_count,0);}return entry;}
voidkgsl_memdesc_init(structkgsl_device*device,structkgsl_memdesc*memdesc,uint64_t flags){structkgsl_mmu*mmu =&device->mmu;unsignedint align;// 初始化kgsl_memdescmemset(memdesc,0,sizeof(*memdesc));/* Turn off SVM if the system doesn't support it */// 判断是否支持KGSL_MMU_IOPGTABLEif(!kgsl_mmu_is_perprocess(mmu))
flags &=~((uint64_t) KGSL_MEMFLAGS_USE_CPU_MAP);/* Secure memory disables advanced addressing modes */if(flags & KGSL_MEMFLAGS_SECURE)
flags &=~((uint64_t) KGSL_MEMFLAGS_USE_CPU_MAP);/* Disable IO coherence if it is not supported on the chip */// 判断是否支持I/O coherencyif(!kgsl_mmu_has_feature(device, KGSL_MMU_IO_COHERENT)){
flags &=~((uint64_t) KGSL_MEMFLAGS_IOCOHERENT);WARN_ONCE(IS_ENABLED(CONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT),"I/O coherency is not supported on this target\n");}elseif(IS_ENABLED(CONFIG_QCOM_KGSL_IOCOHERENCY_DEFAULT))
flags |= KGSL_MEMFLAGS_IOCOHERENT;/*
* We can't enable I/O coherency on uncached surfaces because of
* situations where hardware might snoop the cpu caches which can
* have stale data. This happens primarily due to the limitations
* of dma caching APIs available on arm64
*/if(!kgsl_cachemode_is_cached(flags))
flags &=~((u64) KGSL_MEMFLAGS_IOCOHERENT);if(kgsl_mmu_has_feature(device, KGSL_MMU_NEED_GUARD_PAGE)||(flags & KGSL_MEMFLAGS_GUARD_PAGE))
memdesc->priv |= KGSL_MEMDESC_GUARD_PAGE;if(flags & KGSL_MEMFLAGS_SECURE)
memdesc->priv |= KGSL_MEMDESC_SECURE;// 设置标志位
memdesc->flags = flags;// 设置持有该内存的device
memdesc->dev =&device->pdev->dev;// 对齐
align =max_t(unsignedint,kgsl_memdesc_get_align(memdesc),ilog2(PAGE_SIZE));// 设置kgsl_memdesc的对齐标志位kgsl_memdesc_set_align(memdesc, align);spin_lock_init(&memdesc->lock);}
staticint_kgsl_alloc_pages(structkgsl_memdesc*memdesc,
u64 size,structpage***pages,structdevice*dev){int count =0;// 将内存大小转换为页面数量int npages = size >> PAGE_SHIFT;// attempt to allocate physically contiguous memory by kmalloc// but upon failure, fall back to non-contiguous (vmalloc) allocationstructpage**local =kvcalloc(npages,sizeof(*local), GFP_KERNEL);
u32 page_size, align;
u64 len = size;if(!local)return-ENOMEM;// 共享内存设置成功或者未配置CONFIG_QCOM_KGSL_USE_SHMEM则返回0[见2.2.6.1节]
count =kgsl_memdesc_file_setup(memdesc, size);if(count){kvfree(local);return count;}/* Start with 1MB alignment to get the biggest page we can */
align =ilog2(SZ_1M);// 根据内存大小计算页面大小
page_size =kgsl_get_page_size(len, align);while(len){// 调用kgsl_pool_alloc_page分配, 并将获取的page通过local数组返回int ret =kgsl_alloc_page(&page_size,&local[count],
npages,&align, count, memdesc->shmem_filp, dev);if(ret ==-EAGAIN)continue;elseif(ret <=0){int i;for(i =0; i < count;){int n =1<<compound_order(local[i]);kgsl_free_page(local[i]);
i += n;}kvfree(local);if(!kgsl_sharedmem_noretry_flag)pr_err_ratelimited("kgsl: out of memory: only allocated %lldKb of %lldKb requested\n",(size - len)>>10, size >>10);if(memdesc->shmem_filp)fput(memdesc->shmem_filp);return-ENOMEM;}
count += ret;
npages -= ret;
len -= page_size;
page_size =kgsl_get_page_size(len, align);}// pages作为返回值*pages = local;return count;}
/*
* Attach the memory object to a process by (possibly) getting a GPU address and
* (possibly) mapping it
*/staticintkgsl_mem_entry_attach_and_map(structkgsl_device*device,structkgsl_process_private*process,structkgsl_mem_entry*entry){structkgsl_memdesc*memdesc =&entry->memdesc;int ret;// 2.2.7.1
ret =kgsl_mem_entry_attach_to_process(device, process, entry);if(ret)return ret;if(memdesc->gpuaddr){/*
* Map the memory if a GPU address is already assigned, either
* through kgsl_mem_entry_attach_to_process() or via some other
* SVM process
*/
ret =kgsl_mmu_map(memdesc->pagetable, memdesc);if(ret){kgsl_mem_entry_detach_process(entry);return ret;}}kgsl_memfree_purge(memdesc->pagetable, memdesc->gpuaddr,
memdesc->size);return ret;}
2.2.7.1 kgsl_mem_entry_attach_to_process
staticintkgsl_mem_entry_attach_to_process(structkgsl_device*device,structkgsl_process_private*process,structkgsl_mem_entry*entry){structkgsl_memdesc*memdesc =&entry->memdesc;int ret, id;// kgsl_process_private引用计数加1
ret =kgsl_process_private_get(process);if(!ret)return-EBADF;/* Assign a gpu address */// 判断是否使用与CPU同样的虚拟映射[见2.2.7.2节]以及是否支持IOMMUif(!kgsl_memdesc_use_cpu_map(memdesc)&&kgsl_mmu_get_mmutype(device)!= KGSL_MMU_TYPE_NONE){// GPU页表[见2.2.7.3节]structkgsl_pagetable*pagetable;// 获取进程页表
pagetable =kgsl_memdesc_is_secured(memdesc)?
device->mmu.securepagetable : process->pagetable;// 分配GPU虚拟地址[见2.2.7.4节]
ret =kgsl_mmu_get_gpuaddr(pagetable, memdesc);if(ret){kgsl_process_private_put(process);return ret;}}idr_preload(GFP_KERNEL);spin_lock(&process->mem_lock);/* Allocate the ID but don't attach the pointer just yet */
id =idr_alloc(&process->mem_idr,NULL,1,0, GFP_NOWAIT);spin_unlock(&process->mem_lock);idr_preload_end();if(id <0){if(!kgsl_memdesc_use_cpu_map(memdesc))kgsl_mmu_put_gpuaddr(memdesc->pagetable, memdesc);kgsl_process_private_put(process);return id;}// 更新kgsl_mem_entry的id和kgsl_mem_entry
entry->id = id;
entry->priv = process;return0;}
2.2.7.2 kgsl_memdesc_use_cpu_map
#defineKGSL_MEMFLAGS_USE_CPU_MAP(1ULL<<28)#defineKGSL_MEMFLAGS_SPARSE_PHYS(1ULL<<29)#defineKGSL_MEMFLAGS_SPARSE_VIRT(1ULL<<30)#defineKGSL_MEMFLAGS_IOCOHERENT(1ULL<<31)#defineKGSL_MEMFLAGS_GUARD_PAGE(1ULL<<33)#defineKGSL_MEMFLAGS_VBO(1ULL<<34)/*
* kgsl_memdesc_use_cpu_map - use the same virtual mapping on CPU and GPU?
* @memdesc: the memdesc
*
* Return: true if the memdesc is using SVM mapping
*/// 根据用户空间传入的标志位判断是否在CPU和GPU之间使用同样的虚拟映射staticinline bool
kgsl_memdesc_use_cpu_map(conststructkgsl_memdesc*memdesc){return memdesc &&(memdesc->flags & KGSL_MEMFLAGS_USE_CPU_MAP);}
2.2.7.3 kgsl_pagetable
structkgsl_pagetable{spinlock_t lock;structkref refcount;structlist_head list;unsignedint name;structkobject*kobj;structwork_struct destroy_ws;struct{atomic_t entries;atomic_long_t mapped;atomic_long_t max_mapped;} stats;// conststructkgsl_mmu_pt_ops*pt_ops;uint64_t fault_addr;structkgsl_mmu*mmu;/** @rbtree: all buffers mapped into the pagetable, indexed by gpuaddr */structrb_root rbtree;/** @va_start: Start of virtual range used in this pagetable */
u64 va_start;/** @va_end: End of virtual range */
u64 va_end;/**
* @svm_start: Start of shared virtual memory range. Addresses in this
* range are also valid in the process's CPU address space.
*/
u64 svm_start;/** @svm_end: end of 32 bit compatible range */
u64 svm_end;/**
* @compat_va_start - Start of the "compat" virtual address range for
* forced 32 bit allocations
*/
u64 compat_va_start;/**
* @compat_va_end - End of the "compat" virtual address range for
* forced 32 bit allocations
*/
u64 compat_va_end;
u64 global_base;};
2.2.7.4 kgsl_mmu_get_gpuaddr
#definePT_OP_VALID(_pt, _field)\(((_pt)!=NULL)&&\((_pt)->pt_ops !=NULL)&&\((_pt)->pt_ops->_field !=NULL))/**
* kgsl_mmu_get_gpuaddr - Assign a GPU address to the memdesc
* @pagetable: GPU pagetable to assign the address in
* @memdesc: mem descriptor to assign the memory to
*
* Return: 0 on success or negative on failure
*/staticinlineintkgsl_mmu_get_gpuaddr(structkgsl_pagetable*pagetable,structkgsl_memdesc*memdesc){// 调用kgsl_pagetable->kgsl_mmu_pt_ops-->get_gpuaddr方法分配GPU地址[2.2.7.5节]if(PT_OP_VALID(pagetable, get_gpuaddr))return pagetable->pt_ops->get_gpuaddr(pagetable, memdesc);return-ENOMEM;}
2.2.7.5 kgsl_iommu_get_gpuaddr
staticintkgsl_iommu_get_gpuaddr(structkgsl_pagetable*pagetable,structkgsl_memdesc*memdesc){int ret =0;uint64_t addr, start, end, size;unsignedint align;if(WARN_ON(kgsl_memdesc_use_cpu_map(memdesc)))return-EINVAL;if(memdesc->flags & KGSL_MEMFLAGS_SECURE &&
pagetable->name != KGSL_MMU_SECURE_PT)return-EINVAL;// 获取映射区域(kgsl_memdesc)的大小
size =kgsl_memdesc_footprint(memdesc);
align =max_t(uint64_t,1<<kgsl_memdesc_get_align(memdesc),
PAGE_SIZE);if(memdesc->flags & KGSL_MEMFLAGS_FORCE_32BIT){
start = pagetable->compat_va_start;
end = pagetable->compat_va_end;}else{// Start of virtual range used in this pagetable
start = pagetable->va_start;// End of virtual range
end = pagetable->va_end;}spin_lock(&pagetable->lock);// 获取一块未映射的虚拟地址[2.2.7.6节]
addr =_get_unmapped_area(pagetable, start, end, size, align);if(addr ==(uint64_t)-ENOMEM){
ret =-ENOMEM;goto out;}/*
* This path is only called in a non-SVM path with locks so we can be
* sure we aren't racing with anybody so we don't need to worry about
* taking the lock
*/// 将虚拟地址插入页表[2.2.7.7节]
ret =_insert_gpuaddr(pagetable, addr, size);if(ret ==0){// 设置GPU虚拟地址
memdesc->gpuaddr = addr;// 设置页表
memdesc->pagetable = pagetable;}
out:spin_unlock(&pagetable->lock);return ret;}
2.2.7.6 _get_unmapped_area
/*
* struct kgsl_iommu_addr_entry - entry in the kgsl_pagetable rbtree.
* @base: starting virtual address of the entry
* @size: size of the entry
* @node: the rbtree node
*/structkgsl_iommu_addr_entry{// 起始虚拟地址uint64_t base;uint64_t size;structrb_node node;};staticuint64_t_get_unmapped_area(structkgsl_pagetable*pagetable,uint64_t bottom,uint64_t top,uint64_t size,uint64_t align){// 页表radix tree头节点structrb_node*node =rb_first(&pagetable->rbtree);uint64_t start;
bottom =ALIGN(bottom, align);
start = bottom;while(node !=NULL){uint64_t gap;// 查找rb_node的容器即kgsl_iommu_addr_entrystructkgsl_iommu_addr_entry*entry =rb_entry(node,structkgsl_iommu_addr_entry, node);/*
* Skip any entries that are outside of the range, but make sure
* to account for some that might straddle the lower bound
*/if(entry->base < bottom){if(entry->base + entry->size > bottom)
start =ALIGN(entry->base + entry->size, align);
node =rb_next(node);continue;}/* Stop if we went over the top */if(entry->base >= top)break;/* Make sure there is a gap to consider */if(start < entry->base){
gap = entry->base - start;if(gap >= size)return start;}/* Stop if there is no more room in the region */if(entry->base + entry->size >= top)return(uint64_t)-ENOMEM;/* Start the next cycle at the end of the current entry */
start =ALIGN(entry->base + entry->size, align);
node =rb_next(node);}// 返回起始虚拟地址if(start + size <= top)return start;return(uint64_t)-ENOMEM;}