diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 9cf3725257530..f71704783d795 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -381,6 +381,8 @@ #define X86_FEATURE_HWP_HIGHEST_PERF_CHANGE (14*32+15) /* HWP Highest perf change */ #define X86_FEATURE_HFI (14*32+19) /* "hfi" Hardware Feedback Interface */ +#define X86_FEATURE_DPTI (14*32+24) /* Intel Directed Package Thermal Interrupt */ + /* AMD SVM Feature Identification, CPUID level 0x8000000a (EDX), word 15 */ #define X86_FEATURE_NPT (15*32+ 0) /* "npt" Nested Page Table support */ #define X86_FEATURE_LBRV (15*32+ 1) /* "lbrv" LBR Virtualization support */ diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index d0a0cc8e8bd99..f609d856ef55a 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h @@ -966,6 +966,7 @@ #define THERM_INT_HIGH_ENABLE (1 << 0) #define THERM_INT_LOW_ENABLE (1 << 1) #define THERM_INT_PLN_ENABLE (1 << 24) +#define THERM_INT_DPTI_ENABLE (1 << 25) #define MSR_IA32_THERM_STATUS 0x0000019c @@ -995,6 +996,7 @@ #define PACKAGE_THERM_STATUS_PROCHOT (1 << 0) #define PACKAGE_THERM_STATUS_POWER_LIMIT (1 << 10) +#define PACKAGE_THERM_STATUS_DPTI_ACK (1 << 25) #define PACKAGE_THERM_STATUS_HFI_UPDATED (1 << 26) #define MSR_IA32_PACKAGE_THERM_INTERRUPT 0x000001b2 diff --git a/drivers/thermal/intel/therm_throt.c b/drivers/thermal/intel/therm_throt.c index debc94e2dc169..ec50519971b1c 100644 --- a/drivers/thermal/intel/therm_throt.c +++ b/drivers/thermal/intel/therm_throt.c @@ -14,12 +14,14 @@ * Credits: Adapted from Zwane Mwaikambo's original code in mce_intel.c. * Inspired by Ross Biro's and Al Borchers' counter code. */ +#include #include #include #include #include #include #include +#include #include #include #include @@ -243,16 +245,23 @@ static void thermal_intr_init_pkg_clear_mask(void) * IA32_PACKAGE_THERM_STATUS. */ - /* All bits except BIT 26 depend on CPUID.06H: EAX[6] = 1 */ + /* All bits except BITs 25 and 26 depend on CPUID.06H: EAX[6] = 1 */ if (boot_cpu_has(X86_FEATURE_PTS)) therm_intr_pkg_clear_mask = (BIT(1) | BIT(3) | BIT(5) | BIT(7) | BIT(9) | BIT(11)); /* - * Intel SDM Volume 2A: Thermal and Power Management Leaf + * Intel SDM Volume 1: Thermal and Power Management Leaf * Bit 26: CPUID.06H: EAX[19] = 1 */ if (boot_cpu_has(X86_FEATURE_HFI)) therm_intr_pkg_clear_mask |= BIT(26); + + /* + * Intel SDM Volume 1: Thermal and Power Management Leaf + * Bit 25: CPUID.06H: EAX[24] = 1 + */ + if (boot_cpu_has(X86_FEATURE_DPTI)) + therm_intr_pkg_clear_mask |= BIT(25); } /* @@ -523,13 +532,207 @@ static void thermal_throttle_remove_dev(struct device *dev) sysfs_remove_group(&dev->kobj, &thermal_attr_group); } +static int check_directed_thermal_pkg_intr_ack(void) +{ + unsigned int count = 15000; + u64 msr_val; + + /* + * Hardware acknowledges the directed interrupt setup in 10ms or less. + * Wait 15ms to be safe. + */ + do { + rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val); + udelay(1); + } while (!(msr_val & PACKAGE_THERM_STATUS_DPTI_ACK) && --count); + + if (!count) + return -ETIMEDOUT; + + thermal_clear_package_intr_status(PACKAGE_LEVEL, + PACKAGE_THERM_STATUS_DPTI_ACK); + + return 0; +} + +static void config_directed_thermal_pkg_intr(void *info) +{ + bool enable = *((bool *)info); + u64 msr_val; + + rdmsrl(MSR_IA32_THERM_INTERRUPT, msr_val); + + if (enable) + msr_val |= THERM_INT_DPTI_ENABLE; + else + msr_val &= ~THERM_INT_DPTI_ENABLE; + + wrmsrl(MSR_IA32_THERM_INTERRUPT, msr_val); +} + +/* + * Only accessed from CPU hotplug and syscore callbacks. No extra locking + * needed. + */ +static unsigned int *directed_intr_handler_cpus; + +static bool directed_thermal_pkg_intr_supported(void) +{ + if (!boot_cpu_has(X86_FEATURE_DPTI)) + return false; + + if (!directed_intr_handler_cpus) + return false; + + return true; +} + +/* + * Must be called with cpu_hotplug_lock held to prevent CPUs from going offline + * while iterating through packages. Also, interrupts must be enabled to avoid + * deadlocks in SMP function calls. + * + * The syscore resume callback may call this function but CPU hotplug is disabled + * in that context. It also runs with interrupts disabled, but no SMP function + * calls are issued because the directed interrupt was torn down before suspend. + */ +static void disable_all_directed_thermal_pkg_intr(void) +{ + bool enable = false; + int i; + + if (!directed_thermal_pkg_intr_supported()) + return; + + for (i = 0; i < topology_max_packages(); i++) { + if (directed_intr_handler_cpus[i] == nr_cpu_ids) + continue; + + smp_call_function_single(directed_intr_handler_cpus[i], + config_directed_thermal_pkg_intr, + &enable, true); + } + + kfree(directed_intr_handler_cpus); + directed_intr_handler_cpus = NULL; +} + +static void enable_directed_thermal_pkg_intr(unsigned int cpu) +{ + bool enable = true; + u16 pkg_id; + + if (!directed_thermal_pkg_intr_supported()) + return; + + pkg_id = topology_logical_package_id(cpu); + if (pkg_id >= topology_max_packages()) + return; + + /* Another CPU in this package already handles the directed interrupt. */ + if (directed_intr_handler_cpus[pkg_id] != nr_cpu_ids) + return; + + thermal_clear_package_intr_status(PACKAGE_LEVEL, + PACKAGE_THERM_STATUS_DPTI_ACK); + + config_directed_thermal_pkg_intr(&enable); + if (!check_directed_thermal_pkg_intr_ack()) { + directed_intr_handler_cpus[pkg_id] = cpu; + return; + } + + /* + * A failure indicates faulty hardware. Roll back completely so that + * no other CPU tries. This is especially important during boot as all + * CPUs may come online and would otherwise keep trying. + */ + enable = false; + config_directed_thermal_pkg_intr(&enable); + + disable_all_directed_thermal_pkg_intr(); + + pr_info_once("Failed to direct package thermal interrupts. All CPUs will receive it.\n"); +} + +static void disable_directed_thermal_pkg_intr(unsigned int cpu) +{ + unsigned int new_cpu; + bool enable; + u16 pkg_id; + + if (!directed_thermal_pkg_intr_supported()) + return; + + pkg_id = topology_logical_package_id(cpu); + if (pkg_id >= topology_max_packages()) + return; + + /* Not the CPU handling the directed interrupt. */ + if (directed_intr_handler_cpus[pkg_id] != cpu) + return; + + /* + * The package-level interrupt must remain directed after this CPU goes + * offline. + */ + new_cpu = cpumask_any_but(topology_core_cpumask(cpu), cpu); + if (new_cpu < nr_cpu_ids) { + enable = true; + thermal_clear_package_intr_status(PACKAGE_LEVEL, + PACKAGE_THERM_STATUS_DPTI_ACK); + + /* + * We are here via CPU hotplug. Since we are holding the + * cpu_hotplug_lock, @new_cpu cannot go offline and interrupts + * are enabled, so the SMP function call is safe. + * + * The syscore suspend callback runs with interrupts disabled, + * but it does not reach this path because all the secondary + * CPUs are offline. + */ + smp_call_function_single(new_cpu, config_directed_thermal_pkg_intr, + &enable, true); + } + + /* + * If hardware does not acknowledge the directed interrupt setup on + * @new_cpu, disable the redirection. Since no other CPU is configured + * to receive the package-level interrupt, all CPUs in the package will + * receive it. + */ + enable = false; + if (new_cpu < nr_cpu_ids && check_directed_thermal_pkg_intr_ack()) { + smp_call_function_single(new_cpu, config_directed_thermal_pkg_intr, + &enable, true); + + pr_warn_once("Failed to redirect package thermal interrupt from CPU%u to CPU%u; reverting to broadcast.\n", + cpu, new_cpu); + + new_cpu = nr_cpu_ids; + } + + /* + * Clear the directed interrupt on @cpu. Hardware acknowledgment can be + * ignored since @cpu is going offline. + */ + config_directed_thermal_pkg_intr(&enable); + + directed_intr_handler_cpus[pkg_id] = (new_cpu < nr_cpu_ids) ? new_cpu : nr_cpu_ids; +} + /* Get notified when a cpu comes on/off. Be hotplug friendly. */ static int thermal_throttle_online(unsigned int cpu) { struct thermal_state *state = &per_cpu(thermal_state, cpu); struct device *dev = get_cpu_device(cpu); + int err; u32 l; + err = thermal_throttle_add_dev(dev, cpu); + if (err) + return err; + state->package_throttle.level = PACKAGE_LEVEL; state->core_throttle.level = CORE_LEVEL; @@ -543,11 +746,13 @@ static int thermal_throttle_online(unsigned int cpu) */ intel_hfi_online(cpu); + enable_directed_thermal_pkg_intr(cpu); + /* Unmask the thermal vector after the above workqueues are initialized. */ l = apic_read(APIC_LVTTHMR); apic_write(APIC_LVTTHMR, l & ~APIC_LVT_MASKED); - return thermal_throttle_add_dev(dev, cpu); + return err; } static int thermal_throttle_offline(unsigned int cpu) @@ -560,6 +765,8 @@ static int thermal_throttle_offline(unsigned int cpu) l = apic_read(APIC_LVTTHMR); apic_write(APIC_LVTTHMR, l | APIC_LVT_MASKED); + disable_directed_thermal_pkg_intr(cpu); + intel_hfi_offline(cpu); cancel_delayed_work_sync(&state->package_throttle.therm_work); @@ -572,6 +779,56 @@ static int thermal_throttle_offline(unsigned int cpu) return 0; } +/* + * CPU0 may be handling the directed interrupt, but the CPU hotplug callbacks + * are not called for CPU0 during suspend and resume. + */ +static void directed_pkg_intr_syscore_resume(void *data) +{ + enable_directed_thermal_pkg_intr(0); +} + +static int directed_pkg_intr_syscore_suspend(void *data) +{ + disable_directed_thermal_pkg_intr(0); + + return 0; +} + +static void directed_pkg_intr_syscore_shutdown(void *data) +{ + disable_all_directed_thermal_pkg_intr(); +} + +static const struct syscore_ops directed_pkg_intr_pm_ops = { + .resume = directed_pkg_intr_syscore_resume, + .suspend = directed_pkg_intr_syscore_suspend, + .shutdown = directed_pkg_intr_syscore_shutdown, +}; + +static struct syscore directed_pkg_intr_pm = { + .ops = &directed_pkg_intr_pm_ops, +}; + +static __init void init_directed_pkg_intr(void) +{ + int i; + + if (!boot_cpu_has(X86_FEATURE_DPTI)) + return; + + directed_intr_handler_cpus = kmalloc_array(topology_max_packages(), + sizeof(*directed_intr_handler_cpus), + GFP_KERNEL); + if (!directed_intr_handler_cpus) + return; + + for (i = 0; i < topology_max_packages(); i++) + directed_intr_handler_cpus[i] = nr_cpu_ids; + + register_syscore(&directed_pkg_intr_pm); +} + static __init int thermal_throttle_init_device(void) { int ret; @@ -579,12 +836,19 @@ static __init int thermal_throttle_init_device(void) if (!atomic_read(&therm_throt_en)) return 0; + init_directed_pkg_intr(); + intel_hfi_init(); ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/therm:online", thermal_throttle_online, thermal_throttle_offline); - return ret < 0 ? ret : 0; + if (ret >= 0) + return 0; + + disable_all_directed_thermal_pkg_intr(); + + return ret; } device_initcall(thermal_throttle_init_device);