Xen-hypervisor中SPI中断虚拟化的实现-gic的初始化

介绍

xen支持arm64的type 1的Hypervisor。
wiki: https://wiki.xenproject.org/wiki/Main_Page
github:https://github.com/topics/xen
研究了下SPI中断虚拟化的代码实现
基础知识
《Arm Generic Interrupt Controller v3 and v4 - Virtualization》
https://blog.csdn.net/weixin_47191387/article/details/130188662
《Arm Generic Interrupt Controller v3 and v4》
https://blog.csdn.net/weixin_47191387/article/details/130330239

涉及文件

/xen/arch/arm/arm64/entry.S
/xen/arch/arm/setup.c
/xen/arch/arm/arm64/traps.c
/xen/arch/arm/gic.c
/xen/arch/arm/vgic.c
/xen/arch/arm/gic-v3.c
/xen/arch/arm/gic-vgic.c
/xen/arch/arm/irq.c
/xen/arch/arm/vgic-v3.c
/xen/arch/arm/domain_build.c

gic的初始化

start_xen相当于linux中的start_kernel函数。
start_xen中主要调用了下面三个函数来初始化xen的中断子系统
–>init_traps ：el2的trap相关的系统寄存器的设置
–>gic_preinit ：初始化gic_hw_ops成gicv3 ops
–>gic_init ：gicv3的初始化

init_traps

实现如下

void init_traps(void)
{
    /*
     * Setup Hyp vector base. Note they might get updated with the
     * branch predictor hardening.
     */
    WRITE_SYSREG((vaddr_t)hyp_traps_vector, VBAR_EL2);//VBAR_EL2中的地址设置为hyp_traps_vectord的中断向量表

    /* Trap Debug and Performance Monitor accesses */
    WRITE_SYSREG(HDCR_TDRA|HDCR_TDOSA|HDCR_TDA|HDCR_TPM|HDCR_TPMCR,
                 MDCR_EL2);//Monitor Debug Configuration Register调试相关的先忽略

    /* Trap CP15 c15 used for implementation defined registers */
    WRITE_SYSREG(HSTR_T(15), HSTR_EL2);//Hypervisor System Trap Register

    WRITE_SYSREG(get_default_cptr_flags(), CPTR_EL2);//Architectural Feature Trap Register (EL2)

    /*
     * Configure HCR_EL2 with the bare minimum to run Xen until a guest
     * is scheduled. {A,I,F}MO bits are set to allow EL2 receiving
     * interrupts.
     */
    WRITE_SYSREG(HCR_AMO | HCR_FMO | HCR_IMO, HCR_EL2);//设置HCR_EL2的serror，irq，fiq路由到el2的中断向量表。
    注：关于irq，fiq路由到el2还是el3，在SCR_EL3, Secure Configuration Register寄存器中的IRQ, bit [1]和FIQ, bit [2]也有配置，这两个寄存器同时配置应该以SCR_EL3的配置为准。
    isb();
}

gic_preinit

gicv3_ops的实现如下

static const struct gic_hw_operations gicv3_ops = {
    .info                = &gicv3_info,//gicv3的info
    .init                = gicv3_init,//gicv3的初始化
    .save_state          = gicv3_save_state,//虚拟化vcpu的上下文切换，保存ICH_VMCR_EL2和ICC_SRE_EL1寄存器。
    .restore_state       = gicv3_restore_state,//虚拟化vcpu的上下文切换，加载ICH_VMCR_EL2和ICC_SRE_EL1寄存器。
    .dump_state          = gicv3_dump_state,//dump某个vcpu的list register
    .gic_host_irq_type   = &gicv3_host_irq_type,//用作host irq的desc.handler的ops
    .gic_guest_irq_type  = &gicv3_guest_irq_type,//用作guest irq的desc.handler的ops
    .eoi_irq             = gicv3_eoi_irq,//中断EOI（通过设置ICC_EOIR1_EL1寄存器）同时降低priority（从Running priority到idle priority）
    .deactivate_irq      = gicv3_dir_irq,//deactivate中断，中断变成inactive状态
    .read_irq            = gicv3_read_irq,//中断ack读取ICC_IAR1_EL1寄存器
    .set_active_state    = gicv3_set_active_state,//设置中断状态为active
    .set_pending_state   = gicv3_set_pending_state,//设置中断状态为pending
    .set_irq_type        = gicv3_set_irq_type,//中断类型设置边沿触发或者电平触发
    .set_irq_priority    = gicv3_set_irq_priority,//设置中断的优先级
    .send_SGI            = gicv3_send_sgi,//发送SGI
    .disable_interface   = gicv3_disable_interface,//关闭per-CPU GIC interface
    .update_lr           = gicv3_update_lr,//更新当前cpu的list register中的一个
    .update_hcr_status   = gicv3_hcr_status,//设置ICH_HCR_EL2寄存器
    .clear_lr            = gicv3_clear_lr,//清除当前cpu的list register中的一个
    .read_lr             = gicv3_read_lr,//读取当前cpu的list register中的一个
    .write_lr            = gicv3_write_lr,//写当前cpu的list register中的一个
    .read_vmcr_priority  = gicv3_read_vmcr_priority,//读取ICH_VMCR_EL2中priority的值
    .read_apr            = gicv3_read_apr,//读取ICH_AP1R<n>_EL2寄存器，EL2的group 1 虚拟活动优先级
    .read_pending_state  = gicv3_read_pending_state,//读取中断的pending状态-寄存器GICD_ISPENDR
    .secondary_init      = gicv3_secondary_cpu_init,//secondary cpu启动的时候，percpu相关的的gicv3初始化
    .make_hwdom_dt_node  = gicv3_make_hwdom_dt_node,//处理dt
#ifdef CONFIG_ACPI //假设我们用dt没有用acpi
    .make_hwdom_madt     = gicv3_make_hwdom_madt,
    .get_hwdom_extra_madt_size = gicv3_get_hwdom_extra_madt_size,
#endif
    .iomem_deny_access   = gicv3_iomem_deny_access,//acpi用
    .do_LPI              = gicv3_do_LPI,//处理LPI中断
};

gicv3_ops的注册

void register_gic_ops(const struct gic_hw_operations *ops)
{
    gic_hw_ops = ops;
}

static int __init gicv3_dt_preinit(struct dt_device_node *node, const void *data)
{
    gicv3_info.hw_version = GIC_V3;
    gicv3_info.node = node;
    register_gic_ops(&gicv3_ops);//注册到全局变量gic_hw_ops
    dt_irq_xlate = gic_irq_xlate;

    return 0;
}

DT_DEVICE_START(gicv3, "GICv3", DEVICE_GIC)//gicv3的device
        .dt_match = gicv3_dt_match,
        .init = gicv3_dt_preinit,
DT_DEVICE_END

start_xen会调用gic_preinit

void __init gic_preinit(void)
{
    if ( acpi_disabled )//假设我们用dt没有用acpi
        gic_dt_preinit();
    else
        gic_acpi_preinit();
}

static void __init gic_dt_preinit(void)
{
    int rc;
    struct dt_device_node *node;
    uint8_t num_gics = 0;

    dt_for_each_device_node( dt_host, node )
    {
        if ( !dt_get_property(node, "interrupt-controller", NULL) )
            continue;

        if ( !dt_get_parent(node) )
            continue;

        rc = device_init(node, DEVICE_GIC, NULL);//device init的时候初始化了gicv3的device
        if ( !rc )
        {
            /* NOTE: Only one GIC is supported */
            num_gics = 1;
            break;
        }
    }
    if ( !num_gics )
        panic("Unable to find compatible GIC in the device tree\n");

    /* Set the GIC as the primary interrupt controller */
    dt_interrupt_controller = node;
    dt_device_set_used_by(node, DOMID_XEN);
}

gic_init

start_xen调用gic_init函数

/* Set up the GIC */
void __init gic_init(void)
{
    if ( gic_hw_ops->init() )//调用preinit注册的gicv3_init函数
        panic("Failed to initialize the GIC drivers\n");
    /* Clear LR mask for cpu0 */
    clear_cpu_lr_mask();//清除当前cpu的lr_mask
}

gicv3_init实现主要调用以下函数
–>gicv3_dt_init ：//dt的解析
–>gicv3_dist_init：gicv3对Distributor的设置。
–>gicv3_its_init：gicv3 its的初始化，本文不涉及。
–>gicv3_cpu_init：gicv3关于per-cpu的设置，主要有Redistributor和 CPU interface。
–>gicv3_hyp_init：gicv3关于per-cpu的hypervisor的设置。

gicv3_dt_init主要是对dt中的distributor和redistributor做解析。

gicv3对Distributor的设置实现如下：

static void __init gicv3_dist_init(void)
{
    uint32_t type;
    uint64_t affinity;
    unsigned int nr_lines;
    int i;

    /* Disable the distributor */
    writel_relaxed(0, GICD + GICD_CTLR);//关闭distributor

    type = readl_relaxed(GICD + GICD_TYPER);//读取distributor中的GICD_TYPER寄存器
    nr_lines = 32 * ((type & GICD_TYPE_LINES) + 1);//读取ITLinesNumber位，查看支持的SPI数量

    if ( type & GICD_TYPE_LPIS )//查看是不是支持LPI
        gicv3_lpi_init_host_lpis(GICD_TYPE_ID_BITS(type));

    /* Only 1020 interrupts are supported */
    nr_lines = min(1020U, nr_lines);//spi最大的支持数量为1020
    gicv3_info.nr_lines = nr_lines;

    printk("GICv3: %d lines, (IID %8.8x).\n",
           nr_lines, readl_relaxed(GICD + GICD_IIDR));

    /* Default all global IRQs to level, active low */
    for ( i = NR_GIC_LOCAL_IRQS; i < nr_lines; i += 16 )
        writel_relaxed(0, GICD + GICD_ICFGR + (i / 16) * 4);//设置默认的spi触发方式为电平触发level-sensitive.

    /* Default priority for global interrupts */
    for ( i = NR_GIC_LOCAL_IRQS; i < nr_lines; i += 4 )
        writel_relaxed(GIC_PRI_IRQ_ALL, GICD + GICD_IPRIORITYR + (i / 4) * 4);//设置spi的默认priority

    /* Disable/deactivate all global interrupts */
    for ( i = NR_GIC_LOCAL_IRQS; i < nr_lines; i += 32 )
    {
        writel_relaxed(0xffffffffU, GICD + GICD_ICENABLER + (i / 32) * 4);//设置spi默认为disable
        writel_relaxed(0xffffffffU, GICD + GICD_ICACTIVER + (i / 32) * 4);//设置spi默认为inactive
    }

    /*
     * Configure SPIs as non-secure Group-1. This will only matter
     * if the GIC only has a single security state.
     */
    for ( i = NR_GIC_LOCAL_IRQS; i < nr_lines; i += 32 )
        writel_relaxed(GENMASK(31, 0), GICD + GICD_IGROUPR + (i / 32) * 4);//设置spi默认为non-secure Group-1

    gicv3_dist_wait_for_rwp();

    /* Turn on the distributor */
    writel_relaxed(GICD_CTL_ENABLE | GICD_CTLR_ARE_NS |
                GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1, GICD + GICD_CTLR);//设置GICD_CTLR寄存器。打开Group 0中断，打开Non-secure Group 1中断，设置Affinity routing enabled for Secure state。Non-secure的Affinity Routing没有打开。
          
    /* Route all global IRQs to this CPU */
    affinity = gicv3_mpidr_to_affinity(smp_processor_id());
    /* Make sure we don't broadcast the interrupt */
    affinity &= ~GICD_IROUTER_SPI_MODE_ANY;

    for ( i = NR_GIC_LOCAL_IRQS; i < nr_lines; i++ )
        writeq_relaxed_non_atomic(affinity, GICD + GICD_IROUTER + i * 8);//默认所有spi中断都路由到当前cpu
}

gicv3_cpu_init的实现如下，这里面关于Redistributor的配置主要用于SGI，PPI和LPI，本文不涉及，主要看CPU interface的配置：

static int gicv3_cpu_init(void)
{
    int i, ret;

    /* Register ourselves with the rest of the world */
    if ( gicv3_populate_rdist() )//Redistributor的设置本文不涉及
        return -ENODEV;

    if ( gicv3_enable_redist() )//Redistributor的设置本文不涉及
        return -ENODEV;

    /* If the host has any ITSes, enable LPIs now. */
    if ( gicv3_its_host_has_its() )//its的支持本文不涉及
    {
        if ( !gicv3_enable_lpis() )
            return -EBUSY;
        ret = gicv3_its_setup_collection(smp_processor_id());
        if ( ret )
            return ret;
    }

    /* Set priority on PPI and SGI interrupts */
    for (i = 0; i < NR_GIC_SGI; i += 4)
        writel_relaxed(GIC_PRI_IPI_ALL,
                GICD_RDIST_SGI_BASE + GICR_IPRIORITYR0 + (i / 4) * 4);//设置SGI的priority

    for (i = NR_GIC_SGI; i < NR_GIC_LOCAL_IRQS; i += 4)
        writel_relaxed(GIC_PRI_IRQ_ALL,
                GICD_RDIST_SGI_BASE + GICR_IPRIORITYR0 + (i / 4) * 4);//设置PPI的priority

    /*
     * The activate state is unknown at boot, so make sure all
     * SGIs and PPIs are de-activated.
     */
    writel_relaxed(0xffffffffU, GICD_RDIST_SGI_BASE + GICR_ICACTIVER0);
    /*
     * Disable all PPI interrupts, ensure all SGI interrupts are
     * enabled.
     */
    writel_relaxed(0xffff0000U, GICD_RDIST_SGI_BASE + GICR_ICENABLER0);
    writel_relaxed(0x0000ffffU, GICD_RDIST_SGI_BASE + GICR_ISENABLER0);
    /* Configure SGIs/PPIs as non-secure Group-1 */
    writel_relaxed(GENMASK(31, 0), GICD_RDIST_SGI_BASE + GICR_IGROUPR0);

    gicv3_redist_wait_for_rwp();

    /* Enable system registers */
    gicv3_enable_sre();

    /* No priority grouping */
    WRITE_SYSREG(0, ICC_BPR1_EL1);//设置Binary Point Register

    /* Set priority mask register */
    WRITE_SYSREG(DEFAULT_PMR_VALUE, ICC_PMR_EL1);//设置priority mask register

    /* EOI drops priority, DIR deactivates the interrupt (mode 1) */
    WRITE_SYSREG(GICC_CTLR_EL1_EOImode_drop, ICC_CTLR_EL1);//设置EOI drops priority, DIR deactivates the interrupt

    /* Enable Group1 interrupts */
    WRITE_SYSREG(1, ICC_IGRPEN1_EL1);//当前安全状态下启用Group 1中断

    /* Sync at once at the end of cpu interface configuration */
    isb();

    return 0;
}

gicv3_hyp_init的实现如下

static void gicv3_hyp_init(void)
{
    register_t vtr;

    vtr = READ_SYSREG(ICH_VTR_EL2);//读取ICH_VTR_EL2寄存器
    gicv3_info.nr_lrs  = (vtr & ICH_VTR_NRLRGS) + 1;//支持的List registers 数量
    gicv3.nr_priorities = ((vtr >> ICH_VTR_PRIBITS_SHIFT) &//获取priorities的位数
                          ICH_VTR_PRIBITS_MASK) + 1;

    if ( !((gicv3.nr_priorities > 4) && (gicv3.nr_priorities < 8)) )//priorities的位数大于4位小于8位
        panic("GICv3: Invalid number of priority bits\n");

    WRITE_SYSREG(ICH_VMCR_EOI | ICH_VMCR_VENG1, ICH_VMCR_EL2);//ICH_VMCR_EL2(Interrupt Controller Virtual Machine Control Register)。ICH_VMCR_VENG1是设置Virtual Group 1 interrupt enable。ICH_VMCR_EOI设置Virtual EOI的模式。
    WRITE_SYSREG(GICH_HCR_EN, ICH_HCR_EL2);//设置ICH_HCR_EL2(Interrupt Controller Hyp Control Register)的第0位，使能virtual CPU interface。
}

secondary cpu init的时候(启动cpu的时候)也要做per-cpu的gicv3_cpu_init和gicv3_hyp_init

/* Set up the per-CPU parts of the GIC for a secondary CPU */
static int gicv3_secondary_cpu_init(void)
{
    int res;

    spin_lock(&gicv3.lock);

    res = gicv3_cpu_init();
    if ( res )
        goto out;

    gicv3_hyp_init();

out:
    spin_unlock(&gicv3.lock);

    return res;
}

关于寄存器的配置请查看另一个文章
Xen-hypervisor中SPI中断虚拟化的实现-重要的寄存器

domain中virq的初始化

结构体

struct domain
{
    domid_t          domain_id;
    ......
    struct vcpu    **vcpu;
    ......
    struct arch_domain arch;
    ......
}

struct arch_domain
{
#ifdef CONFIG_ARM_64
    enum domain_type type;
#endif
   ......
   struct vgic_dist vgic;
   ........
}

struct vgic_dist {
    /* Version of the vGIC */
    enum gic_version version;
    /* GIC HW version specific vGIC driver handler */
    const struct vgic_ops *handler;//注册成v3_ops
    /*
     * Covers access to other members of this struct _except_ for
     * shared_irqs where each member contains its own locking.
     *
     * If both class of lock is required then this lock must be
     * taken first. If multiple rank locks are required (including
     * the per-vcpu private_irqs rank) then they must be taken in
     * rank order.
     */
    spinlock_t lock;
    uint32_t ctlr;//GICD_CTLR寄存器的备份
    int nr_spis; /* Number of SPIs */
    unsigned long *allocated_irqs; /* bitmap of IRQs allocated */
    struct vgic_irq_rank *shared_irqs;//domain上的spi的rank数组指针
    /*
     * SPIs are domain global, SGIs and PPIs are per-VCPU and stored in
     * struct arch_vcpu.
     */
    struct pending_irq *pending_irqs;//domian上每个spi中断申请一个pending_irq
    /* Base address for guest GIC */
    paddr_t dbase; /* Distributor base address */
    paddr_t cbase; /* CPU interface base address */
    paddr_t csize; /* CPU interface size */
    paddr_t vbase; /* Virtual CPU interface base address */
#ifdef CONFIG_GICV3
    /* GIC V3 addressing */
    /* List of contiguous occupied by the redistributors */
    struct vgic_rdist_region {
        paddr_t base;                   /* Base address */
        paddr_t size;                   /* Size */
        unsigned int first_cpu;         /* First CPU handled */
    } *rdist_regions;
    int nr_regions;                     /* Number of rdist regions */
    unsigned long int nr_lpis;
    uint64_t rdist_propbase;
    struct rb_root its_devices;         /* Devices mapped to an ITS */
    spinlock_t its_devices_lock;        /* Protects the its_devices tree */
    struct radix_tree_root pend_lpi_tree; /* Stores struct pending_irq's */
    rwlock_t pend_lpi_tree_lock;        /* Protects the pend_lpi_tree */
    struct list_head vits_list;         /* List of virtual ITSes */
    unsigned int intid_bits;
    /*
     * TODO: if there are more bool's being added below, consider
     * a flags variable instead.
     */
    bool rdists_enabled;                /* Is any redistributor enabled? */
    bool has_its;
#endif
};

static const struct vgic_ops v3_ops = {
    .vcpu_init   = vgic_v3_vcpu_init,//获取vcpu的Redistributors的base地址，添加到vcpu的结构体
    .domain_init = vgic_v3_domain_init,//1.获取当前domain所有vcpu的Redistributors的base地址，以及distributors的base地址 2.domain的its初始化 3.为Distributor注册mmio句柄 4.为redistributors占用的每个连续区域注册mmio处理程序。
    .domain_free = vgic_v3_domain_free,
    .emulate_reg  = vgic_v3_emulate_reg,
    .lpi_to_pending = vgic_v3_lpi_to_pending,//根据虚拟LPI编号，在pending的LPI树中查找LPI
    .lpi_get_priority = vgic_v3_lpi_get_priority,//根据虚拟LPI编号，在pending的LPI树中查找LPI，并返回priority
};

//vgic_v3_domain_init中为Distributor注册mmio句柄
static const struct mmio_handler_ops vgic_distr_mmio_handler = {
    .read  = vgic_v3_distr_mmio_read,//通过iomap对gic的Distributor的读
    .write = vgic_v3_distr_mmio_write,//通过iomap对gic的Distributor的写
};

每个rank中支持32个中断，所以priority和vcpu都是32个元素

struct vgic_irq_rank {
    spinlock_t lock; /* Covers access to all other members of this struct */

    uint8_t index;//rank的index

    uint32_t ienable;//rank上enable virq的mask
    uint32_t icfg[2];

    /*
     * Provide efficient access to the priority of an vIRQ while keeping
     * the emulation simple.
     * Note, this is working fine as long as Xen is using little endian.
     */
    union {
        uint8_t priority[32];//提供对vIRQ优先级的有效访问
        uint32_t ipriorityr[8];
    };

    /*
     * It's more convenient to store a target VCPU per vIRQ
     * than the register ITARGETSR/IROUTER itself.
     * Use atomic operations to read/write the vcpu fields to avoid
     * taking the rank lock.
     */
    //每个vIRQ的目标VCPU设置，比使用寄存器ITARGETSR/IROUTER本身更方便。
    uint8_t vcpu[32];
};

pending_irq定义的宏代表
GIC_IRQ_GUEST_QUEUED：//inject一个virq，且没有添加到list register寄存器的状态。
GIC_IRQ_GUEST_ACTIVE：//virq的active状态
GIC_IRQ_GUEST_VISIBLE：//添加到list register寄存器的状态
GIC_IRQ_GUEST_ENABLED：//virq使能标志
GIC_IRQ_GUEST_MIGRATING：暂时先不考虑MIGRATING的情况
GIC_IRQ_GUEST_PRISTINE_LPI：关于LPI的本文不涉及

struct pending_irq
{
#define GIC_IRQ_GUEST_QUEUED   0
#define GIC_IRQ_GUEST_ACTIVE   1
#define GIC_IRQ_GUEST_VISIBLE  2
#define GIC_IRQ_GUEST_ENABLED  3
#define GIC_IRQ_GUEST_MIGRATING   4
#define GIC_IRQ_GUEST_PRISTINE_LPI  5
    unsigned long status;   //status状态参看上面的宏
    struct irq_desc *desc; /* only set if the irq corresponds to a physical irq */
    unsigned int irq;  //irq或者virq中断号
#define GIC_INVALID_LR         (uint8_t)~0
    uint8_t lr;      //当前pending_irq占用的list register的number
    uint8_t priority;    //当前pending_irq的优先级
    uint8_t lpi_priority;       /* Caches the priority if this is an LPI. */
    uint8_t lpi_vcpu_id;        /* The VCPU for an LPI. */
    /* inflight is used to append instances of pending_irq to
     * vgic.inflight_irqs */
    struct list_head inflight; 
    /* lr_queue is used to append instances of pending_irq to
     * lr_pending. lr_pending is a per vcpu queue, therefore lr_queue
     * accesses are protected with the vgic lock.
     * TODO: when implementing irq migration, taking only the current
     * vgic lock is not going to be enough. */
    struct list_head lr_queue;//将当前的pending_irq挂在per vcpu的lr_pending上
};

调用流程

domain_create
–>arch_domain_create
---->domain_vgic_register
---->domain_vgic_init

domain_vgic_register主要注册了vgic_ops到domain.arch.vgic的handler

int domain_vgic_register(struct domain *d, int *mmio_count)
{
    switch ( d->arch.vgic.version )
    {
#ifdef CONFIG_GICV3
    case GIC_V3:
        if ( vgic_v3_init(d, mmio_count) )//默认适用gicv3
           return -ENODEV;
        break;
#endif
    case GIC_V2:
        if ( vgic_v2_init(d, mmio_count) )
            return -ENODEV;
        break;
    default:
        printk(XENLOG_G_ERR "d%d: Unknown vGIC version %u\n",
               d->domain_id, d->arch.vgic.version);
        return -ENODEV;
    }

    return 0;
}

int vgic_v3_init(struct domain *d, int *mmio_count)
{
    if ( !vgic_v3_hw.enabled )
    {
        printk(XENLOG_G_ERR
               "d%d: vGICv3 is not supported on this platform.\n",
               d->domain_id);
        return -ENODEV;
    }

    /* GICD region + number of Redistributors */
    *mmio_count = vgic_v3_max_rdist_count(d) + 1;

    /* one region per ITS */
    *mmio_count += vgic_v3_its_count(d);

    register_vgic_ops(d, &v3_ops);//d->arch.vgic.handler初始化为v3_ops

    return 0;
}

domain.arch.vgic的初始化

int domain_vgic_init(struct domain *d, unsigned int nr_spis)
{
    int i;
    int ret;

    d->arch.vgic.ctlr = 0;//ctlr的值默认为0

    /*
     * The vGIC relies on having a pending_irq available for every IRQ
     * described in the ranks. As each rank describes 32 interrupts, we
     * need to make sure the number of SPIs is a multiple of 32.
     */
    nr_spis = ROUNDUP(nr_spis, 32);

    /* Limit the number of virtual SPIs supported to (1020 - 32) = 988  */
    if ( nr_spis > (1020 - NR_LOCAL_IRQS) )
        return -EINVAL;

    d->arch.vgic.nr_spis = nr_spis;//spi的数量

    spin_lock_init(&d->arch.vgic.lock);

    d->arch.vgic.shared_irqs =//domain上的spi中断的rank数组指针
        xzalloc_array(struct vgic_irq_rank, DOMAIN_NR_RANKS(d));
    if ( d->arch.vgic.shared_irqs == NULL )
        return -ENOMEM;

    d->arch.vgic.pending_irqs =//为每个spi中断申请一个pending_irq
        xzalloc_array(struct pending_irq, d->arch.vgic.nr_spis);
    if ( d->arch.vgic.pending_irqs == NULL )
        return -ENOMEM;

    for (i=0; i<d->arch.vgic.nr_spis; i++)
        vgic_init_pending_irq(&d->arch.vgic.pending_irqs[i], i + 32);//初始化pending_irq数组的变量设置p->irq = virq（i + 32）
    /* SPIs are routed to VCPU0 by default */
    for ( i = 0; i < DOMAIN_NR_RANKS(d); i++ )//设置rank的index，以及默认路由到VCPU0
        vgic_rank_init(&d->arch.vgic.shared_irqs[i], i + 1, 0);

    ret = d->arch.vgic.handler->domain_init(d);//1.获取当前domain所有vcpu的Redistributors的base地址，以及distributors的base地址 2.domain的its初始化 3.为Distributor注册mmio句柄 4.为redistributors占用的每个连续区域注册mmio处理程序。
    if ( ret )
        return ret;

    d->arch.vgic.allocated_irqs =//分配中断bitmap的内存申请
        xzalloc_array(unsigned long, BITS_TO_LONGS(vgic_num_irqs(d)));
    if ( !d->arch.vgic.allocated_irqs )
        return -ENOMEM;

    /* vIRQ0-15 (SGIs) are reserved */
    for ( i = 0; i < NR_GIC_SGI; i++ )//SGI默认配置成allocated
        set_bit(i, d->arch.vgic.allocated_irqs);

    return 0;
}

vcpu上的vgic初始化

结构体

struct vcpu
{
    int              vcpu_id;
    ......
    struct arch_vcpu arch;
}

struct arch_vcpu
{
    ......
    struct vgic_cpu vgic;
    ......
}

struct vgic_cpu {
    /*
     * SGIs and PPIs are per-VCPU, SPIs are domain global and in
     * struct arch_domain.
     */
    struct pending_irq pending_irqs[32];//sgi和ppi的pending_irq结构体
    struct vgic_irq_rank *private_irqs;//irq_rank管理当前vcpu的私有(SGIs/PPIs)中断

    /* This list is ordered by IRQ priority and it is used to keep
     * track of the IRQs that the VGIC injected into the guest.
     * Depending on the availability of LR registers, the IRQs might
     * actually be in an LR, and therefore injected into the guest,
     * or queued in gic.lr_pending.
     * As soon as an IRQ is EOI'd by the guest and removed from the
     * corresponding LR it is also removed from this list. */
    //此列表按IRQ优先级排序，用于跟踪VGIC注入到guest中的IRQ。根据LR寄存器的可用性，IRQ实际上可能在LR中，因此可能被注入到guest中，或者在gic.LR_pending中排队。一旦IRQ被guest EOI’d并从相应的LR中删除，它也会从该列表中删除。
    struct list_head inflight_irqs;
    /* lr_pending is used to queue IRQs (struct pending_irq) that the
     * vgic tried to inject in the guest (calling gic_raise_guest_irq) but
     * no LRs were available at the time.
     * As soon as an LR is freed we remove the first IRQ from this
     * list and write it to the LR register.
     * lr_pending is a subset of vgic.inflight_irqs. */
    //lr_pending用于对vgic试图注入到guest中的irq（struct pending_irq）进行排队（调用gic_raise_guest_irq），但当时没有可用的lr。一旦LR被释放，我们就从这个列表中删除第一个IRQ，并将其写入LR寄存器。lr_pending是vgic.inflight_irqs的子集。
    struct list_head lr_pending;
    spinlock_t lock;

    /* GICv3: redistributor base and flags for this vCPU */
    paddr_t rdist_base;//per-vcpu的redistributor base
    uint64_t rdist_pendbase;
#define VGIC_V3_RDIST_LAST      (1 << 0)        /* last vCPU of the rdist */
#define VGIC_V3_LPIS_ENABLED    (1 << 1)
    uint8_t flags;
};

调用流程

vcpu_create
–>arch_vcpu_create
---->vcpu_vgic_init

int vcpu_vgic_init(struct vcpu *v)
{
    int i;

    v->arch.vgic.private_irqs = xzalloc(struct vgic_irq_rank);//一个rank支持32个中断，够一个vcpu的sgi和ppi用了
    if ( v->arch.vgic.private_irqs == NULL )
      return -ENOMEM;

    /* SGIs/PPIs are always routed to this VCPU */
    vgic_rank_init(v->arch.vgic.private_irqs, 0, v->vcpu_id);//rank的index设置为0，路由到当前vcpu

    v->domain->arch.vgic.handler->vcpu_init(v);//设置当前vcpu的Redistributors的base地址

    memset(&v->arch.vgic.pending_irqs, 0, sizeof(v->arch.vgic.pending_irqs));
    for (i = 0; i < 32; i++)//设置pending_irq结构体的irq变量为0-31（sgi和ppi）
        vgic_init_pending_irq(&v->arch.vgic.pending_irqs[i], i);

    INIT_LIST_HEAD(&v->arch.vgic.inflight_irqs);
    INIT_LIST_HEAD(&v->arch.vgic.lr_pending);
    spin_lock_init(&v->arch.vgic.lock);

    return 0;
}