| /* SPDX-License-Identifier: GPL-2.0-only */ | |
| /* | |
| * | |
| * Derived from book3s_hv_rmhandlers.S, which is: | |
| * | |
| * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> | |
| */ | |
| #include <asm/reg.h> | |
| #include <asm/ppc_asm.h> | |
| #include <asm/asm-offsets.h> | |
| #include <asm/export.h> | |
| #include <asm/tm.h> | |
| #include <asm/cputable.h> | |
| #ifdef CONFIG_PPC_TRANSACTIONAL_MEM | |
| #define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM) | |
| /* | |
| * Save transactional state and TM-related registers. | |
| * Called with: | |
| * - r3 pointing to the vcpu struct | |
| * - r4 containing the MSR with current TS bits: | |
| * (For HV KVM, it is VCPU_MSR ; For PR KVM, it is host MSR). | |
| * - r5 containing a flag indicating that non-volatile registers | |
| * must be preserved. | |
| * If r5 == 0, this can modify all checkpointed registers, but | |
| * restores r1, r2 before exit. If r5 != 0, this restores the | |
| * MSR TM/FP/VEC/VSX bits to their state on entry. | |
| */ | |
| _GLOBAL(__kvmppc_save_tm) | |
| mflr r0 | |
| std r0, PPC_LR_STKOFF(r1) | |
| stdu r1, -SWITCH_FRAME_SIZE(r1) | |
| mr r9, r3 | |
| cmpdi cr7, r5, 0 | |
| /* Turn on TM. */ | |
| mfmsr r8 | |
| mr r10, r8 | |
| li r0, 1 | |
| rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG | |
| ori r8, r8, MSR_FP | |
| oris r8, r8, (MSR_VEC | MSR_VSX)@h | |
| mtmsrd r8 | |
| rldicl. r4, r4, 64 - MSR_TS_S_LG, 62 | |
| beq 1f /* TM not active in guest. */ | |
| std r1, HSTATE_SCRATCH2(r13) | |
| std r3, HSTATE_SCRATCH1(r13) | |
| /* Save CR on the stack - even if r5 == 0 we need to get cr7 back. */ | |
| mfcr r6 | |
| SAVE_GPR(6, r1) | |
| /* Save DSCR so we can restore it to avoid running with user value */ | |
| mfspr r7, SPRN_DSCR | |
| SAVE_GPR(7, r1) | |
| /* | |
| * We are going to do treclaim., which will modify all checkpointed | |
| * registers. Save the non-volatile registers on the stack if | |
| * preservation of non-volatile state has been requested. | |
| */ | |
| beq cr7, 3f | |
| SAVE_NVGPRS(r1) | |
| /* MSR[TS] will be 0 (non-transactional) once we do treclaim. */ | |
| li r0, 0 | |
| rldimi r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG | |
| SAVE_GPR(10, r1) /* final MSR value */ | |
| 3: | |
| #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE | |
| BEGIN_FTR_SECTION | |
| /* Emulation of the treclaim instruction needs TEXASR before treclaim */ | |
| mfspr r6, SPRN_TEXASR | |
| std r6, VCPU_ORIG_TEXASR(r3) | |
| END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_HV_ASSIST) | |
| #endif | |
| /* Clear the MSR RI since r1, r13 are all going to be foobar. */ | |
| li r5, 0 | |
| mtmsrd r5, 1 | |
| li r3, TM_CAUSE_KVM_RESCHED | |
| /* All GPRs are volatile at this point. */ | |
| TRECLAIM(R3) | |
| /* Temporarily store r13 and r9 so we have some regs to play with */ | |
| SET_SCRATCH0(r13) | |
| GET_PACA(r13) | |
| std r9, PACATMSCRATCH(r13) | |
| ld r9, HSTATE_SCRATCH1(r13) | |
| /* Save away PPR soon so we don't run with user value. */ | |
| std r0, VCPU_GPRS_TM(0)(r9) | |
| mfspr r0, SPRN_PPR | |
| HMT_MEDIUM | |
| /* Reload stack pointer. */ | |
| std r1, VCPU_GPRS_TM(1)(r9) | |
| ld r1, HSTATE_SCRATCH2(r13) | |
| /* Set MSR RI now we have r1 and r13 back. */ | |
| std r2, VCPU_GPRS_TM(2)(r9) | |
| li r2, MSR_RI | |
| mtmsrd r2, 1 | |
| /* Reload TOC pointer. */ | |
| ld r2, PACATOC(r13) | |
| /* Save all but r0-r2, r9 & r13 */ | |
| reg = 3 | |
| .rept 29 | |
| .if (reg != 9) && (reg != 13) | |
| std reg, VCPU_GPRS_TM(reg)(r9) | |
| .endif | |
| reg = reg + 1 | |
| .endr | |
| /* ... now save r13 */ | |
| GET_SCRATCH0(r4) | |
| std r4, VCPU_GPRS_TM(13)(r9) | |
| /* ... and save r9 */ | |
| ld r4, PACATMSCRATCH(r13) | |
| std r4, VCPU_GPRS_TM(9)(r9) | |
| /* Restore host DSCR and CR values, after saving guest values */ | |
| mfcr r6 | |
| mfspr r7, SPRN_DSCR | |
| stw r6, VCPU_CR_TM(r9) | |
| std r7, VCPU_DSCR_TM(r9) | |
| REST_GPR(6, r1) | |
| REST_GPR(7, r1) | |
| mtcr r6 | |
| mtspr SPRN_DSCR, r7 | |
| /* Save away checkpointed SPRs. */ | |
| std r0, VCPU_PPR_TM(r9) | |
| mflr r5 | |
| mfctr r7 | |
| mfspr r8, SPRN_AMR | |
| mfspr r10, SPRN_TAR | |
| mfxer r11 | |
| std r5, VCPU_LR_TM(r9) | |
| std r7, VCPU_CTR_TM(r9) | |
| std r8, VCPU_AMR_TM(r9) | |
| std r10, VCPU_TAR_TM(r9) | |
| std r11, VCPU_XER_TM(r9) | |
| /* Save FP/VSX. */ | |
| addi r3, r9, VCPU_FPRS_TM | |
| bl store_fp_state | |
| addi r3, r9, VCPU_VRS_TM | |
| bl store_vr_state | |
| mfspr r6, SPRN_VRSAVE | |
| stw r6, VCPU_VRSAVE_TM(r9) | |
| /* Restore non-volatile registers if requested to */ | |
| beq cr7, 1f | |
| REST_NVGPRS(r1) | |
| REST_GPR(10, r1) | |
| 1: | |
| /* | |
| * We need to save these SPRs after the treclaim so that the software | |
| * error code is recorded correctly in the TEXASR. Also the user may | |
| * change these outside of a transaction, so they must always be | |
| * context switched. | |
| */ | |
| mfspr r7, SPRN_TEXASR | |
| std r7, VCPU_TEXASR(r9) | |
| mfspr r5, SPRN_TFHAR | |
| mfspr r6, SPRN_TFIAR | |
| std r5, VCPU_TFHAR(r9) | |
| std r6, VCPU_TFIAR(r9) | |
| /* Restore MSR state if requested */ | |
| beq cr7, 2f | |
| mtmsrd r10, 0 | |
| 2: | |
| addi r1, r1, SWITCH_FRAME_SIZE | |
| ld r0, PPC_LR_STKOFF(r1) | |
| mtlr r0 | |
| blr | |
| /* | |
| * _kvmppc_save_tm_pr() is a wrapper around __kvmppc_save_tm(), so that it can | |
| * be invoked from C function by PR KVM only. | |
| */ | |
| _GLOBAL(_kvmppc_save_tm_pr) | |
| mflr r0 | |
| std r0, PPC_LR_STKOFF(r1) | |
| stdu r1, -PPC_MIN_STKFRM(r1) | |
| mfspr r8, SPRN_TAR | |
| std r8, PPC_MIN_STKFRM-8(r1) | |
| li r5, 1 /* preserve non-volatile registers */ | |
| bl __kvmppc_save_tm | |
| ld r8, PPC_MIN_STKFRM-8(r1) | |
| mtspr SPRN_TAR, r8 | |
| addi r1, r1, PPC_MIN_STKFRM | |
| ld r0, PPC_LR_STKOFF(r1) | |
| mtlr r0 | |
| blr | |
| EXPORT_SYMBOL_GPL(_kvmppc_save_tm_pr); | |
| /* | |
| * Restore transactional state and TM-related registers. | |
| * Called with: | |
| * - r3 pointing to the vcpu struct. | |
| * - r4 is the guest MSR with desired TS bits: | |
| * For HV KVM, it is VCPU_MSR | |
| * For PR KVM, it is provided by caller | |
| * - r5 containing a flag indicating that non-volatile registers | |
| * must be preserved. | |
| * If r5 == 0, this potentially modifies all checkpointed registers, but | |
| * restores r1, r2 from the PACA before exit. | |
| * If r5 != 0, this restores the MSR TM/FP/VEC/VSX bits to their state on entry. | |
| */ | |
| _GLOBAL(__kvmppc_restore_tm) | |
| mflr r0 | |
| std r0, PPC_LR_STKOFF(r1) | |
| cmpdi cr7, r5, 0 | |
| /* Turn on TM/FP/VSX/VMX so we can restore them. */ | |
| mfmsr r5 | |
| mr r10, r5 | |
| li r6, MSR_TM >> 32 | |
| sldi r6, r6, 32 | |
| or r5, r5, r6 | |
| ori r5, r5, MSR_FP | |
| oris r5, r5, (MSR_VEC | MSR_VSX)@h | |
| mtmsrd r5 | |
| /* | |
| * The user may change these outside of a transaction, so they must | |
| * always be context switched. | |
| */ | |
| ld r5, VCPU_TFHAR(r3) | |
| ld r6, VCPU_TFIAR(r3) | |
| ld r7, VCPU_TEXASR(r3) | |
| mtspr SPRN_TFHAR, r5 | |
| mtspr SPRN_TFIAR, r6 | |
| mtspr SPRN_TEXASR, r7 | |
| mr r5, r4 | |
| rldicl. r5, r5, 64 - MSR_TS_S_LG, 62 | |
| beq 9f /* TM not active in guest */ | |
| /* Make sure the failure summary is set, otherwise we'll program check | |
| * when we trechkpt. It's possible that this might have been not set | |
| * on a kvmppc_set_one_reg() call but we shouldn't let this crash the | |
| * host. | |
| */ | |
| oris r7, r7, (TEXASR_FS)@h | |
| mtspr SPRN_TEXASR, r7 | |
| /* | |
| * Make a stack frame and save non-volatile registers if requested. | |
| */ | |
| stdu r1, -SWITCH_FRAME_SIZE(r1) | |
| std r1, HSTATE_SCRATCH2(r13) | |
| mfcr r6 | |
| mfspr r7, SPRN_DSCR | |
| SAVE_GPR(2, r1) | |
| SAVE_GPR(6, r1) | |
| SAVE_GPR(7, r1) | |
| beq cr7, 4f | |
| SAVE_NVGPRS(r1) | |
| /* MSR[TS] will be 1 (suspended) once we do trechkpt */ | |
| li r0, 1 | |
| rldimi r10, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG | |
| SAVE_GPR(10, r1) /* final MSR value */ | |
| 4: | |
| /* | |
| * We need to load up the checkpointed state for the guest. | |
| * We need to do this early as it will blow away any GPRs, VSRs and | |
| * some SPRs. | |
| */ | |
| mr r31, r3 | |
| addi r3, r31, VCPU_FPRS_TM | |
| bl load_fp_state | |
| addi r3, r31, VCPU_VRS_TM | |
| bl load_vr_state | |
| mr r3, r31 | |
| lwz r7, VCPU_VRSAVE_TM(r3) | |
| mtspr SPRN_VRSAVE, r7 | |
| ld r5, VCPU_LR_TM(r3) | |
| lwz r6, VCPU_CR_TM(r3) | |
| ld r7, VCPU_CTR_TM(r3) | |
| ld r8, VCPU_AMR_TM(r3) | |
| ld r9, VCPU_TAR_TM(r3) | |
| ld r10, VCPU_XER_TM(r3) | |
| mtlr r5 | |
| mtcr r6 | |
| mtctr r7 | |
| mtspr SPRN_AMR, r8 | |
| mtspr SPRN_TAR, r9 | |
| mtxer r10 | |
| /* | |
| * Load up PPR and DSCR values but don't put them in the actual SPRs | |
| * till the last moment to avoid running with userspace PPR and DSCR for | |
| * too long. | |
| */ | |
| ld r29, VCPU_DSCR_TM(r3) | |
| ld r30, VCPU_PPR_TM(r3) | |
| /* Clear the MSR RI since r1, r13 are all going to be foobar. */ | |
| li r5, 0 | |
| mtmsrd r5, 1 | |
| /* Load GPRs r0-r28 */ | |
| reg = 0 | |
| .rept 29 | |
| ld reg, VCPU_GPRS_TM(reg)(r31) | |
| reg = reg + 1 | |
| .endr | |
| mtspr SPRN_DSCR, r29 | |
| mtspr SPRN_PPR, r30 | |
| /* Load final GPRs */ | |
| ld 29, VCPU_GPRS_TM(29)(r31) | |
| ld 30, VCPU_GPRS_TM(30)(r31) | |
| ld 31, VCPU_GPRS_TM(31)(r31) | |
| /* TM checkpointed state is now setup. All GPRs are now volatile. */ | |
| TRECHKPT | |
| /* Now let's get back the state we need. */ | |
| HMT_MEDIUM | |
| GET_PACA(r13) | |
| ld r1, HSTATE_SCRATCH2(r13) | |
| REST_GPR(7, r1) | |
| mtspr SPRN_DSCR, r7 | |
| /* Set the MSR RI since we have our registers back. */ | |
| li r5, MSR_RI | |
| mtmsrd r5, 1 | |
| /* Restore TOC pointer and CR */ | |
| REST_GPR(2, r1) | |
| REST_GPR(6, r1) | |
| mtcr r6 | |
| /* Restore non-volatile registers if requested to. */ | |
| beq cr7, 5f | |
| REST_GPR(10, r1) | |
| REST_NVGPRS(r1) | |
| 5: addi r1, r1, SWITCH_FRAME_SIZE | |
| ld r0, PPC_LR_STKOFF(r1) | |
| mtlr r0 | |
| 9: /* Restore MSR bits if requested */ | |
| beqlr cr7 | |
| mtmsrd r10, 0 | |
| blr | |
| /* | |
| * _kvmppc_restore_tm_pr() is a wrapper around __kvmppc_restore_tm(), so that it | |
| * can be invoked from C function by PR KVM only. | |
| */ | |
| _GLOBAL(_kvmppc_restore_tm_pr) | |
| mflr r0 | |
| std r0, PPC_LR_STKOFF(r1) | |
| stdu r1, -PPC_MIN_STKFRM(r1) | |
| /* save TAR so that it can be recovered later */ | |
| mfspr r8, SPRN_TAR | |
| std r8, PPC_MIN_STKFRM-8(r1) | |
| li r5, 1 | |
| bl __kvmppc_restore_tm | |
| ld r8, PPC_MIN_STKFRM-8(r1) | |
| mtspr SPRN_TAR, r8 | |
| addi r1, r1, PPC_MIN_STKFRM | |
| ld r0, PPC_LR_STKOFF(r1) | |
| mtlr r0 | |
| blr | |
| EXPORT_SYMBOL_GPL(_kvmppc_restore_tm_pr); | |
| #endif /* CONFIG_PPC_TRANSACTIONAL_MEM */ |