diff options
79 files changed, 4399 insertions, 1549 deletions
diff --git a/Documentation/ABI/testing/sysfs-power b/Documentation/ABI/testing/sysfs-power index 50b368d490b5..f523e5a3ac33 100644 --- a/Documentation/ABI/testing/sysfs-power +++ b/Documentation/ABI/testing/sysfs-power @@ -7,30 +7,35 @@ Description: subsystem. What: /sys/power/state -Date: May 2014 +Date: November 2016 Contact: Rafael J. Wysocki <rjw@rjwysocki.net> Description: The /sys/power/state file controls system sleep states. Reading from this file returns the available sleep state - labels, which may be "mem", "standby", "freeze" and "disk" - (hibernation). The meanings of the first three labels depend on - the relative_sleep_states command line argument as follows: - 1) relative_sleep_states = 1 - "mem", "standby", "freeze" represent non-hibernation sleep - states from the deepest ("mem", always present) to the - shallowest ("freeze"). "standby" and "freeze" may or may - not be present depending on the capabilities of the - platform. "freeze" can only be present if "standby" is - present. - 2) relative_sleep_states = 0 (default) - "mem" - "suspend-to-RAM", present if supported. - "standby" - "power-on suspend", present if supported. - "freeze" - "suspend-to-idle", always present. - - Writing to this file one of these strings causes the system to - transition into the corresponding state, if available. See - Documentation/power/states.txt for a description of what - "suspend-to-RAM", "power-on suspend" and "suspend-to-idle" mean. + labels, which may be "mem" (suspend), "standby" (power-on + suspend), "freeze" (suspend-to-idle) and "disk" (hibernation). + + Writing one of the above strings to this file causes the system + to transition into the corresponding state, if available. + + See Documentation/power/states.txt for more information. + +What: /sys/power/mem_sleep +Date: November 2016 +Contact: Rafael J. Wysocki <rjw@rjwysocki.net> +Description: + The /sys/power/mem_sleep file controls the operating mode of + system suspend. Reading from it returns the available modes + as "s2idle" (always present), "shallow" and "deep" (present if + supported). The mode that will be used on subsequent attempts + to suspend the system (by writing "mem" to the /sys/power/state + file described above) is enclosed in square brackets. + + Writing one of the above strings to this file causes the mode + represented by it to be used on subsequent attempts to suspend + the system. + + See Documentation/power/states.txt for more information. What: /sys/power/disk Date: September 2006 diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 62d68b2056de..be2d6d0a03a4 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -1560,6 +1560,12 @@ disable Do not enable intel_pstate as the default scaling driver for the supported processors + passive + Use intel_pstate as a scaling driver, but configure it + to work with generic cpufreq governors (instead of + enabling its internal governor). This mode cannot be + used along with the hardware-managed P-states (HWP) + feature. force Enable intel_pstate on systems that prohibit it by default in favor of acpi-cpufreq. Forcing the intel_pstate driver @@ -1580,6 +1586,9 @@ Description Table, specifies preferred power management profile as "Enterprise Server" or "Performance Server", then this feature is turned on by default. + per_cpu_perf_limits + Allow per-logical-CPU P-State performance control limits using + cpufreq sysfs interface intremap= [X86-64, Intel-IOMMU] on enable Interrupt Remapping (default) @@ -2122,6 +2131,12 @@ memory contents and reserves bad memory regions that are detected. + mem_sleep_default= [SUSPEND] Default system suspend mode: + s2idle - Suspend-To-Idle + shallow - Power-On Suspend or equivalent (if supported) + deep - Suspend-To-RAM or equivalent (if supported) + See Documentation/power/states.txt. + meye.*= [HW] Set MotionEye Camera parameters See Documentation/video4linux/meye.txt. @@ -3475,13 +3490,6 @@ [KNL, SMP] Set scheduler's default relax_domain_level. See Documentation/cgroup-v1/cpusets.txt. - relative_sleep_states= - [SUSPEND] Use sleep state labeling where the deepest - state available other than hibernation is always "mem". - Format: { "0" | "1" } - 0 -- Traditional sleep state labels. - 1 -- Relative sleep state labels. - reserve= [KNL,BUGS] Force the kernel to ignore some iomem area reservetop= [X86-32] diff --git a/Documentation/cpu-freq/cpufreq-stats.txt b/Documentation/cpu-freq/cpufreq-stats.txt index 8d9773f23550..3c355f6ad834 100644 --- a/Documentation/cpu-freq/cpufreq-stats.txt +++ b/Documentation/cpu-freq/cpufreq-stats.txt @@ -44,11 +44,17 @@ the stats driver insertion. total 0 drwxr-xr-x 2 root root 0 May 14 16:06 . drwxr-xr-x 3 root root 0 May 14 15:58 .. +--w------- 1 root root 4096 May 14 16:06 reset -r--r--r-- 1 root root 4096 May 14 16:06 time_in_state -r--r--r-- 1 root root 4096 May 14 16:06 total_trans -r--r--r-- 1 root root 4096 May 14 16:06 trans_table -------------------------------------------------------------------------------- +- reset +Write-only attribute that can be used to reset the stat counters. This can be +useful for evaluating system behaviour under different governors without the +need for a reboot. + - time_in_state This gives the amount of time spent in each of the frequencies supported by this CPU. The cat output will have "<frequency> <time>" pair in each line, which diff --git a/Documentation/cpu-freq/intel-pstate.txt b/Documentation/cpu-freq/intel-pstate.txt index e6bd1e6512a5..1953994ef5e6 100644 --- a/Documentation/cpu-freq/intel-pstate.txt +++ b/Documentation/cpu-freq/intel-pstate.txt @@ -48,7 +48,7 @@ In addition to the frequency-controlling interfaces provided by the cpufreq core, the driver provides its own sysfs files to control the P-State selection. These files have been added to /sys/devices/system/cpu/intel_pstate/. Any changes made to these files are applicable to all CPUs (even in a -multi-package system). +multi-package system, Refer to later section on placing "Per-CPU limits"). max_perf_pct: Limits the maximum P-State that will be requested by the driver. It states it as a percentage of the available performance. The @@ -120,13 +120,57 @@ frequency is fictional for Intel Core processors. Even if the scaling driver selects a single P-State, the actual frequency the processor will run at is selected by the processor itself. +Per-CPU limits + +The kernel command line option "intel_pstate=per_cpu_perf_limits" forces +the intel_pstate driver to use per-CPU performance limits. When it is set, +the sysfs control interface described above is subject to limitations. +- The following controls are not available for both read and write + /sys/devices/system/cpu/intel_pstate/max_perf_pct + /sys/devices/system/cpu/intel_pstate/min_perf_pct +- The following controls can be used to set performance limits, as far as the +architecture of the processor permits: + /sys/devices/system/cpu/cpu*/cpufreq/scaling_max_freq + /sys/devices/system/cpu/cpu*/cpufreq/scaling_min_freq + /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +- User can still observe turbo percent and number of P-States from + /sys/devices/system/cpu/intel_pstate/turbo_pct + /sys/devices/system/cpu/intel_pstate/num_pstates +- User can read write system wide turbo status + /sys/devices/system/cpu/no_turbo + +Support of energy performance hints +It is possible to provide hints to the HWP algorithms in the processor +to be more performance centric to more energy centric. When the driver +is using HWP, two additional cpufreq sysfs attributes are presented for +each logical CPU. +These attributes are: + - energy_performance_available_preferences + - energy_performance_preference + +To get list of supported hints: +$ cat energy_performance_available_preferences + default performance balance_performance balance_power power + +The current preference can be read or changed via cpufreq sysfs +attribute "energy_performance_preference". Reading from this attribute +will display current effective setting. User can write any of the valid +preference string to this attribute. User can always restore to power-on +default by writing "default". + +Since threads can migrate to different CPUs, this is possible that the +new CPU may have different energy performance preference than the previous +one. To avoid such issues, either threads can be pinned to specific CPUs +or set the same energy performance preference value to all CPUs. + Tuning Intel P-State driver -When HWP mode is not used, debugfs files have also been added to allow the -tuning of the internal governor algorithm. These files are located at -/sys/kernel/debug/pstate_snb/. The algorithm uses a PID (Proportional -Integral Derivative) controller. The PID tunable parameters are: +When the performance can be tuned using PID (Proportional Integral +Derivative) controller, debugfs files are provided for adjusting performance. +They are presented under: +/sys/kernel/debug/pstate_snb/ +The PID tunable parameters are: deadband d_gain_pct i_gain_pct diff --git a/Documentation/devicetree/bindings/cpufreq/brcm,stb-avs-cpu-freq.txt b/Documentation/devicetree/bindings/cpufreq/brcm,stb-avs-cpu-freq.txt new file mode 100644 index 000000000000..af2385795d78 --- /dev/null +++ b/Documentation/devicetree/bindings/cpufreq/brcm,stb-avs-cpu-freq.txt @@ -0,0 +1,78 @@ +Broadcom AVS mail box and interrupt register bindings +===================================================== + +A total of three DT nodes are required. One node (brcm,avs-cpu-data-mem) +references the mailbox register used to communicate with the AVS CPU[1]. The +second node (brcm,avs-cpu-l2-intr) is required to trigger an interrupt on +the AVS CPU. The interrupt tells the AVS CPU that it needs to process a +command sent to it by a driver. Interrupting the AVS CPU is mandatory for +commands to be processed. + +The interface also requires a reference to the AVS host interrupt controller, +so a driver can react to interrupts generated by the AVS CPU whenever a command +has been processed. See [2] for more information on the brcm,l2-intc node. + +[1] The AVS CPU is an independent co-processor that runs proprietary +firmware. On some SoCs, this firmware supports DFS and DVFS in addition to +Adaptive Voltage Scaling. + +[2] Documentation/devicetree/bindings/interrupt-controller/brcm,l2-intc.txt + + +Node brcm,avs-cpu-data-mem +-------------------------- + +Required properties: +- compatible: must include: brcm,avs-cpu-data-mem and + should include: one of brcm,bcm7271-avs-cpu-data-mem or + brcm,bcm7268-avs-cpu-data-mem +- reg: Specifies base physical address and size of the registers. +- interrupts: The interrupt that the AVS CPU will use to interrupt the host + when a command completed. +- interrupt-parent: The interrupt controller the above interrupt is routed + through. +- interrupt-names: The name of the interrupt used to interrupt the host. + +Optional properties: +- None + +Node brcm,avs-cpu-l2-intr +------------------------- + +Required properties: +- compatible: must include: brcm,avs-cpu-l2-intr and + should include: one of brcm,bcm7271-avs-cpu-l2-intr or + brcm,bcm7268-avs-cpu-l2-intr +- reg: Specifies base physical address and size of the registers. + +Optional properties: +- None + + +Example +======= + + avs_host_l2_intc: interrupt-controller@f04d1200 { + #interrupt-cells = <1>; + compatible = "brcm,l2-intc"; + interrupt-parent = <&intc>; + reg = <0xf04d1200 0x48>; + interrupt-controller; + interrupts = <0x0 0x19 0x0>; + interrupt-names = "avs"; + }; + + avs-cpu-data-mem@f04c4000 { + compatible = "brcm,bcm7271-avs-cpu-data-mem", + "brcm,avs-cpu-data-mem"; + reg = <0xf04c4000 0x60>; + interrupts = <0x1a>; + interrupt-parent = <&avs_host_l2_intc>; + interrupt-names = "sw_intr"; + }; + + avs-cpu-l2-intr@f04d1100 { + compatible = "brcm,bcm7271-avs-cpu-l2-intr", + "brcm,avs-cpu-l2-intr"; + reg = <0xf04d1100 0x10>; + }; diff --git a/Documentation/devicetree/bindings/opp/opp.txt b/Documentation/devicetree/bindings/opp/opp.txt index ee91cbdd95ee..9f5ca4457b5f 100644 --- a/Documentation/devicetree/bindings/opp/opp.txt +++ b/Documentation/devicetree/bindings/opp/opp.txt @@ -86,8 +86,14 @@ Optional properties: Single entry is for target voltage and three entries are for <target min max> voltages. - Entries for multiple regulators must be present in the same order as - regulators are specified in device's DT node. + Entries for multiple regulators shall be provided in the same field separated + by angular brackets <>. The OPP binding doesn't provide any provisions to + relate the values to their power supplies or the order in which the supplies + need to be configured and that is left for the implementation specific + binding. + + Entries for all regulators shall be of the same size, i.e. either all use a + single value or triplets. - opp-microvolt-<name>: Named opp-microvolt property. This is exactly similar to the above opp-microvolt property, but allows multiple voltage ranges to be @@ -104,10 +110,13 @@ Optional properties: Should only be set if opp-microvolt is set for the OPP. - Entries for multiple regulators must be present in the same order as - regulators are specified in device's DT node. If this property isn't required - for few regulators, then this should be marked as zero for them. If it isn't - required for any regulator, then this property need not be present. + Entries for multiple regulators shall be provided in the same field separated + by angular brackets <>. If current values aren't required for a regulator, + then it shall be filled with 0. If current values aren't required for any of + the regulators, then this field is not required. The OPP binding doesn't + provide any provisions to relate the values to their power supplies or the + order in which the supplies need to be configured and that is left for the + implementation specific binding. - opp-microamp-<name>: Named opp-microamp property. Similar to opp-microvolt-<name> property, but for microamp instead. @@ -386,10 +395,12 @@ Example 4: Handling multiple regulators / { cpus { cpu@0 { - compatible = "arm,cortex-a7"; + compatible = "vendor,cpu-type"; ... - cpu-supply = <&cpu_supply0>, <&cpu_supply1>, <&cpu_supply2>; + vcc0-supply = <&cpu_supply0>; + vcc1-supply = <&cpu_supply1>; + vcc2-supply = <&cpu_supply2>; operating-points-v2 = <&cpu0_opp_table>; }; }; diff --git a/Documentation/devicetree/bindings/power/domain-idle-state.txt b/Documentation/devicetree/bindings/power/domain-idle-state.txt new file mode 100644 index 000000000000..eefc7ed22ca2 --- /dev/null +++ b/Documentation/devicetree/bindings/power/domain-idle-state.txt @@ -0,0 +1,33 @@ +PM Domain Idle State Node: + +A domain idle state node represents the state parameters that will be used to +select the state when there are no active components in the domain. + +The state node has the following parameters - + +- compatible: + Usage: Required + Value type: <string> + Definition: Must be "domain-idle-state". + +- entry-latency-us + Usage: Required + Value type: <prop-encoded-array> + Definition: u32 value representing worst case latency in + microseconds required to enter the idle state. + The exit-latency-us duration may be guaranteed + only after entry-latency-us has passed. + +- exit-latency-us + Usage: Required + Value type: <prop-encoded-array> + Definition: u32 value representing worst case latency + in microseconds required to exit the idle state. + +- min-residency-us + Usage: Required + Value type: <prop-encoded-array> + Definition: u32 value representing minimum residency duration + in microseconds after which the idle state will yield + power benefits after overcoming the overhead in entering +i the idle state. diff --git a/Documentation/devicetree/bindings/power/power_domain.txt b/Documentation/devicetree/bindings/power/power_domain.txt index 025b5e7df61c..723e1ad937da 100644 --- a/Documentation/devicetree/bindings/power/power_domain.txt +++ b/Documentation/devicetree/bindings/power/power_domain.txt @@ -29,6 +29,15 @@ Optional properties: specified by this binding. More details about power domain specifier are available in the next section. +- domain-idle-states : A phandle of an idle-state that shall be soaked into a + generic domain power state. The idle state definitions are + compatible with domain-idle-state specified in [1]. + The domain-idle-state property reflects the idle state of this PM domain and + not the idle states of the devices or sub-domains in the PM domain. Devices + and sub-domains have their own idle-states independent of the parent + domain's idle states. In the absence of this property, the domain would be + considered as capable of being powered-on or powered-off. + Example: power: power-controller@12340000 { @@ -59,6 +68,38 @@ The nodes above define two power controllers: 'parent' and 'child'. Domains created by the 'child' power controller are subdomains of '0' power domain provided by the 'parent' power controller. +Example 3: + parent: power-controller@12340000 { + compatible = "foo,power-controller"; + reg = <0x12340000 0x1000>; + #power-domain-cells = <0>; + domain-idle-states = <&DOMAIN_RET>, <&DOMAIN_PWR_DN>; + }; + + child: power-controller@12341000 { + compatible = "foo,power-controller"; + reg = <0x12341000 0x1000>; + power-domains = <&parent 0>; + #power-domain-cells = <0>; + domain-idle-states = <&DOMAIN_PWR_DN>; + }; + + DOMAIN_RET: state@0 { + compatible = "domain-idle-state"; + reg = <0x0>; + entry-latency-us = <1000>; + exit-latency-us = <2000>; + min-residency-us = <10000>; + }; + + DOMAIN_PWR_DN: state@1 { + compatible = "domain-idle-state"; + reg = <0x1>; + entry-latency-us = <5000>; + exit-latency-us = <8000>; + min-residency-us = <7000>; + }; + ==PM domain consumers== Required properties: @@ -76,3 +117,5 @@ Example: The node above defines a typical PM domain consumer device, which is located inside a PM domain with index 0 of a power controller represented by a node with the label "power". + +[1]. Documentation/devicetree/bindings/power/domain-idle-state.txt diff --git a/Documentation/power/devices.txt b/Documentation/power/devices.txt index 8ba6625fdd63..73ddea39a9ce 100644 --- a/Documentation/power/devices.txt +++ b/Documentation/power/devices.txt @@ -607,7 +607,9 @@ individually. Instead, a set of devices sharing a power resource can be put into a low-power state together at the same time by turning off the shared power resource. Of course, they also need to be put into the full-power state together, by turning the shared power resource on. A set of devices with this -property is often referred to as a power domain. +property is often referred to as a power domain. A power domain may also be +nested inside another power domain. The nested domain is referred to as the +sub-domain of the parent domain. Support for power domains is provided through the pm_domain field of struct device. This field is a pointer to an object of type struct dev_pm_domain, @@ -629,6 +631,16 @@ support for power domains into subsystem-level callbacks, for example by modifying the platform bus type. Other platforms need not implement it or take it into account in any way. +Devices may be defined as IRQ-safe which indicates to the PM core that their +runtime PM callbacks may be invoked with disabled interrupts (see +Documentation/power/runtime_pm.txt for more information). If an IRQ-safe +device belongs to a PM domain, the runtime PM of the domain will be +disallowed, unless the domain itself is defined as IRQ-safe. However, it +makes sense to define a PM domain as IRQ-safe only if all the devices in it +are IRQ-safe. Moreover, if an IRQ-safe domain has a parent domain, the runtime +PM of the parent is only allowed if the parent itself is IRQ-safe too with the +additional restriction that all child domains of an IRQ-safe parent must also +be IRQ-safe. Device Low Power (suspend) States --------------------------------- diff --git a/Documentation/power/states.txt b/Documentation/power/states.txt index 50f3ef9177c1..8a39ce45d8a0 100644 --- a/Documentation/power/states.txt +++ b/Documentation/power/states.txt @@ -8,25 +8,43 @@ for each state. The states are represented by strings that can be read or written to the /sys/power/state file. Those strings may be "mem", "standby", "freeze" and -"disk", where the last one always represents hibernation (Suspend-To-Disk) and -the meaning of the remaining ones depends on the relative_sleep_states command -line argument. - -For relative_sleep_states=1, the strings "mem", "standby" and "freeze" label the -available non-hibernation sleep states from the deepest to the shallowest, -respectively. In that case, "mem" is always present in /sys/power/state, -because there is at least one non-hibernation sleep state in every system. If -the given system supports two non-hibernation sleep states, "standby" is present -in /sys/power/state in addition to "mem". If the system supports three -non-hibernation sleep states, "freeze" will be present in /sys/power/state in -addition to "mem" and "standby". - -For relative_sleep_states=0, which is the default, the following descriptions -apply. - -state: Suspend-To-Idle +"disk", where the last three always represent Power-On Suspend (if supported), +Suspend-To-Idle and hibernation (Suspend-To-Disk), respectively. + +The meaning of the "mem" string is controlled by the /sys/power/mem_sleep file. +It contains strings representing the available modes of system suspend that may +be triggered by writing "mem" to /sys/power/state. These modes are "s2idle" +(Suspend-To-Idle), "shallow" (Power-On Suspend) and "deep" (Suspend-To-RAM). +The "s2idle" mode is always available, while the other ones are only available +if supported by the platform (if not supported, the strings representing them +are not present in /sys/power/mem_sleep). The string representing the suspend +mode to be used subsequently is enclosed in square brackets. Writing one of +the other strings present in /sys/power/mem_sleep to it causes the suspend mode +to be used subsequently to change to the one represented by that string. + +Consequently, there are two ways to cause the system to go into the +Suspend-To-Idle sleep state. The first one is to write "freeze" directly to +/sys/power/state. The second one is to write "s2idle" to /sys/power/mem_sleep +and then to wrtie "mem" to /sys/power/state. Similarly, there are two ways +to cause the system to go into the Power-On Suspend sleep state (the strings to +write to the control files in that case are "standby" or "shallow" and "mem", +respectively) if that state is supported by the platform. In turn, there is +only one way to cause the system to go into the Suspend-To-RAM state (write +"deep" into /sys/power/mem_sleep and "mem" into /sys/power/state). + +The default suspend mode (ie. the one to be used without writing anything into +/sys/power/mem_sleep) is either "deep" (if Suspend-To-RAM is supported) or +"s2idle", but it can be overridden by the value of the "mem_sleep_default" +parameter in the kernel command line. On some ACPI-based systems, depending on +the information in the FADT, the default may be "s2idle" even if Suspend-To-RAM +is supported. + +The properties of all of the sleep states are described below. + + +State: Suspend-To-Idle ACPI state: S0 -Label: "freeze" +Label: "s2idle" ("freeze") This state is a generic, pure software, light-weight, system sleep state. It allows more energy to be saved relative to runtime idle by freezing user @@ -35,13 +53,13 @@ lower-power than available at run time), such that the processors can spend more time in their idle states. This state can be used for platforms without Power-On Suspend/Suspend-to-RAM -support, or it can be used in addition to Suspend-to-RAM (memory sleep) -to provide reduced resume latency. It is always supported. +support, or it can be used in addition to Suspend-to-RAM to provide reduced +resume latency. It is always supported. State: Standby / Power-On Suspend ACPI State: S1 -Label: "standby" +Label: "shallow" ("standby") This state, if supported, offers moderate, though real, power savings, while providing a relatively low-latency transition back to a working system. No @@ -58,7 +76,7 @@ state. State: Suspend-to-RAM ACPI State: S3 -Label: "mem" +Label: "deep" This state, if supported, offers significant power savings as everything in the system is put into a low-power state, except for memory, which should be placed diff --git a/MAINTAINERS b/MAINTAINERS index 3d7d66cdb44c..34ef63763566 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2764,6 +2764,14 @@ L: bcm-kernel-feedback-list@broadcom.com S: Maintained F: drivers/mtd/nand/brcmnand/ +BROADCOM STB AVS CPUFREQ DRIVER +M: Markus Mayer <mmayer@broadcom.com> +M: bcm-kernel-feedback-list@broadcom.com +L: linux-pm@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/cpufreq/brcm,stb-avs-cpu-freq.txt +F: drivers/cpufreq/brcmstb* + BROADCOM SPECIFIC AMBA DRIVER (BCMA) M: RafaÅ‚ MiÅ‚ecki <zajec5@gmail.com> L: linux-wireless@vger.kernel.org @@ -3356,6 +3364,7 @@ L: linux-pm@vger.kernel.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git T: git git://git.linaro.org/people/vireshk/linux.git (For ARM Updates) +B: https://bugzilla.kernel.org F: Documentation/cpu-freq/ F: drivers/cpufreq/ F: include/linux/cpufreq.h @@ -3395,6 +3404,7 @@ M: Daniel Lezcano <daniel.lezcano@linaro.org> L: linux-pm@vger.kernel.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git +B: https://bugzilla.kernel.org F: drivers/cpuidle/* F: include/linux/cpuidle.h @@ -6362,9 +6372,11 @@ S: Maintained F: drivers/platform/x86/intel-vbtn.c INTEL IDLE DRIVER +M: Jacob Pan <jacob.jun.pan@linux.intel.com> M: Len Brown <lenb@kernel.org> L: linux-pm@vger.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git +B: https://bugzilla.kernel.org S: Supported F: drivers/idle/intel_idle.c diff --git a/arch/arm/mach-imx/gpc.c b/arch/arm/mach-imx/gpc.c index b54db47f6f32..1dc2a34b9dbd 100644 --- a/arch/arm/mach-imx/gpc.c +++ b/arch/arm/mach-imx/gpc.c @@ -380,13 +380,6 @@ static struct pu_domain imx6q_pu_domain = { .name = "PU", .power_off = imx6q_pm_pu_power_off, .power_on = imx6q_pm_pu_power_on, - .states = { - [0] = { - .power_off_latency_ns = 25000, - .power_on_latency_ns = 2000000, - }, - }, - .state_count = 1, }, }; @@ -430,6 +423,16 @@ static int imx_gpc_genpd_init(struct device *dev, struct regulator *pu_reg) if (!IS_ENABLED(CONFIG_PM_GENERIC_DOMAINS)) return 0; + imx6q_pu_domain.base.states = devm_kzalloc(dev, + sizeof(*imx6q_pu_domain.base.states), + GFP_KERNEL); + if (!imx6q_pu_domain.base.states) + return -ENOMEM; + + imx6q_pu_domain.base.states[0].power_off_latency_ns = 25000; + imx6q_pu_domain.base.states[0].power_on_latency_ns = 2000000; + imx6q_pu_domain.base.state_count = 1; + for (i = 0; i < ARRAY_SIZE(imx_gpc_domains); i++) pm_genpd_init(imx_gpc_domains[i], NULL, false); diff --git a/arch/x86/kernel/acpi/wakeup_64.S b/arch/x86/kernel/acpi/wakeup_64.S index 169963f471bb..50b8ed0317a3 100644 --- a/arch/x86/kernel/acpi/wakeup_64.S +++ b/arch/x86/kernel/acpi/wakeup_64.S @@ -109,6 +109,15 @@ ENTRY(do_suspend_lowlevel) movq pt_regs_r14(%rax), %r14 movq pt_regs_r15(%rax), %r15 +#ifdef CONFIG_KASAN + /* + * The suspend path may have poisoned some areas deeper in the stack, + * which we now need to unpoison. + */ + movq %rsp, %rdi + call kasan_unpoison_task_stack_below +#endif + xorl %eax, %eax addq $8, %rsp FRAME_END diff --git a/arch/x86/power/hibernate_64.c b/arch/x86/power/hibernate_64.c index 9634557a5444..ded2e8272382 100644 --- a/arch/x86/power/hibernate_64.c +++ b/arch/x86/power/hibernate_64.c @@ -11,6 +11,10 @@ #include <linux/gfp.h> #include <linux/smp.h> #include <linux/suspend.h> +#include <linux/scatterlist.h> +#include <linux/kdebug.h> + +#include <crypto/hash.h> #include <asm/init.h> #include <asm/proto.h> @@ -177,14 +181,86 @@ int pfn_is_nosave(unsigned long pfn) return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn); } +#define MD5_DIGEST_SIZE 16 + struct restore_data_record { unsigned long jump_address; unsigned long jump_address_phys; unsigned long cr3; unsigned long magic; + u8 e820_digest[MD5_DIGEST_SIZE]; }; -#define RESTORE_MAGIC 0x123456789ABCDEF0UL +#define RESTORE_MAGIC 0x23456789ABCDEF01UL + +#if IS_BUILTIN(CONFIG_CRYPTO_MD5) +/** + * get_e820_md5 - calculate md5 according to given e820 map + * + * @map: the e820 map to be calculated + * @buf: the md5 result to be stored to + */ +static int get_e820_md5(struct e820map *map, void *buf) +{ + struct scatterlist sg; + struct crypto_ahash *tfm; + int size; + int ret = 0; + + tfm = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC); + if (IS_ERR(tfm)) + return -ENOMEM; + + { + AHASH_REQUEST_ON_STACK(req, tfm); + size = offsetof(struct e820map, map) + + sizeof(struct e820entry) * map->nr_map; + ahash_request_set_tfm(req, tfm); + sg_init_one(&sg, (u8 *)map, size); + ahash_request_set_callback(req, 0, NULL, NULL); + ahash_request_set_crypt(req, &sg, buf, size); + + if (crypto_ahash_digest(req)) + ret = -EINVAL; + ahash_request_zero(req); + } + crypto_free_ahash(tfm); + + return ret; +} + +static void hibernation_e820_save(void *buf) +{ + get_e820_md5(e820_saved, buf); +} + +static bool hibernation_e820_mismatch(void *buf) +{ + int ret; + u8 result[MD5_DIGEST_SIZE]; + + memset(result, 0, MD5_DIGEST_SIZE); + /* If there is no digest in suspend kernel, let it go. */ + if (!memcmp(result, buf, MD5_DIGEST_SIZE)) + return false; + + ret = get_e820_md5(e820_saved, result); + if (ret) + return true; + + return memcmp(result, buf, MD5_DIGEST_SIZE) ? true : false; +} +#else +static void hibernation_e820_save(void *buf) +{ +} + +static bool hibernation_e820_mismatch(void *buf) +{ + /* If md5 is not builtin for restore kernel, let it go. */ + return false; +} +#endif /** * arch_hibernation_header_save - populate the architecture specific part @@ -201,6 +277,9 @@ int arch_hibernation_header_save(void *addr, unsigned int max_size) rdr->jump_address_phys = __pa_symbol(&restore_registers); rdr->cr3 = restore_cr3; rdr->magic = RESTORE_MAGIC; + + hibernation_e820_save(rdr->e820_digest); + return 0; } @@ -216,5 +295,16 @@ int arch_hibernation_header_restore(void *addr) restore_jump_address = rdr->jump_address; jump_address_phys = rdr->jump_address_phys; restore_cr3 = rdr->cr3; - return (rdr->magic == RESTORE_MAGIC) ? 0 : -EINVAL; + + if (rdr->magic != RESTORE_MAGIC) { + pr_crit("Unrecognized hibernate image header format!\n"); + return -EINVAL; + } + + if (hibernation_e820_mismatch(rdr->e820_digest)) { + pr_crit("Hibernate inconsistent memory map detected!\n"); + return -ENODEV; + } + + return 0; } diff --git a/drivers/acpi/processor_perflib.c b/drivers/acpi/processor_perflib.c index bb01dea39fdc..f0b4a981b8d3 100644 --- a/drivers/acpi/processor_perflib.c +++ b/drivers/acpi/processor_perflib.c @@ -157,7 +157,7 @@ static void acpi_processor_ppc_ost(acpi_handle handle, int status) status, NULL); } -int acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag) +void acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag) { int ret; @@ -168,7 +168,7 @@ int acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag) */ if (event_flag) acpi_processor_ppc_ost(pr->handle, 1); - return 0; + return; } ret = acpi_processor_get_platform_limit(pr); @@ -182,10 +182,8 @@ int acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag) else acpi_processor_ppc_ost(pr->handle, 0); } - if (ret < 0) - return (ret); - else - return cpufreq_update_policy(pr->id); + if (ret >= 0) + cpufreq_update_policy(pr->id); } int acpi_processor_get_bios_limit(int cpu, unsigned int *limit) @@ -465,11 +463,33 @@ int acpi_processor_get_performance_info(struct acpi_processor *pr) return result; } EXPORT_SYMBOL_GPL(acpi_processor_get_performance_info); -int acpi_processor_notify_smm(struct module *calling_module) + +int acpi_processor_pstate_control(void) { acpi_status status; - static int is_done = 0; + if (!acpi_gbl_FADT.smi_command || !acpi_gbl_FADT.pstate_control) + return 0; + + ACPI_DEBUG_PRINT((ACPI_DB_INFO, + "Writing pstate_control [0x%x] to smi_command [0x%x]\n", + acpi_gbl_FADT.pstate_control, acpi_gbl_FADT.smi_command)); + + status = acpi_os_write_port(acpi_gbl_FADT.smi_command, + (u32)acpi_gbl_FADT.pstate_control, 8); + if (ACPI_SUCCESS(status)) + return 1; + + ACPI_EXCEPTION((AE_INFO, status, + "Failed to write pstate_control [0x%x] to smi_command [0x%x]", + acpi_gbl_FADT.pstate_control, acpi_gbl_FADT.smi_command)); + return -EIO; +} + +int acpi_processor_notify_smm(struct module *calling_module) +{ + static int is_done = 0; + int result; if (!(acpi_processor_ppc_status & PPC_REGISTERED)) return -EBUSY; @@ -492,26 +512,15 @@ int acpi_processor_notify_smm(struct module *calling_module) is_done = -EIO; - /* Can't write pstate_control to smi_command if either value is zero */ - if ((!acpi_gbl_FADT.smi_command) || (!acpi_gbl_FADT.pstate_control)) { + result = acpi_processor_pstate_control(); + if (!result) { ACPI_DEBUG_PRINT((ACPI_DB_INFO, "No SMI port or pstate_control\n")); module_put(calling_module); return 0; } - - ACPI_DEBUG_PRINT((ACPI_DB_INFO, - "Writing pstate_control [0x%x] to smi_command [0x%x]\n", - acpi_gbl_FADT.pstate_control, acpi_gbl_FADT.smi_command)); - - status = acpi_os_write_port(acpi_gbl_FADT.smi_command, - (u32) acpi_gbl_FADT.pstate_control, 8); - if (ACPI_FAILURE(status)) { - ACPI_EXCEPTION((AE_INFO, status, - "Failed to write pstate_control [0x%x] to " - "smi_command [0x%x]", acpi_gbl_FADT.pstate_control, - acpi_gbl_FADT.smi_command)); + if (result < 0) { module_put(calling_module); - return status; + return result; } /* Success. If there's no _PPC, we need to fear nothing, so diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c index 54abb26b7366..9b6cebe227a0 100644 --- a/drivers/acpi/sleep.c +++ b/drivers/acpi/sleep.c @@ -674,6 +674,14 @@ static void acpi_sleep_suspend_setup(void) if (acpi_sleep_state_supported(i)) sleep_states[i] = 1; + /* + * Use suspend-to-idle by default if ACPI_FADT_LOW_POWER_S0 is set and + * the default suspend mode was not selected from the command line. + */ + if (acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0 && + mem_sleep_default > PM_SUSPEND_MEM) + mem_sleep_default = PM_SUSPEND_FREEZE; + suspend_set_ops(old_suspend_ordering ? &acpi_suspend_ops_old : &acpi_suspend_ops); freeze_set_ops(&acpi_freeze_ops); diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c index e023066e4215..5711708532db 100644 --- a/drivers/base/power/domain.c +++ b/drivers/base/power/domain.c @@ -39,6 +39,105 @@ static LIST_HEAD(gpd_list); static DEFINE_MUTEX(gpd_list_lock); +struct genpd_lock_ops { + void (*lock)(struct generic_pm_domain *genpd); + void (*lock_nested)(struct generic_pm_domain *genpd, int depth); + int (*lock_interruptible)(struct generic_pm_domain *genpd); + void (*unlock)(struct generic_pm_domain *genpd); +}; + +static void genpd_lock_mtx(struct generic_pm_domain *genpd) +{ + mutex_lock(&genpd->mlock); +} + +static void genpd_lock_nested_mtx(struct generic_pm_domain *genpd, + int depth) +{ + mutex_lock_nested(&genpd->mlock, depth); +} + +static int genpd_lock_interruptible_mtx(struct generic_pm_domain *genpd) +{ + return mutex_lock_interruptible(&genpd->mlock); +} + +static void genpd_unlock_mtx(struct generic_pm_domain *genpd) +{ + return mutex_unlock(&genpd->mlock); +} + +static const struct genpd_lock_ops genpd_mtx_ops = { + .lock = genpd_lock_mtx, + .lock_nested = genpd_lock_nested_mtx, + .lock_interruptible = genpd_lock_interruptible_mtx, + .unlock = genpd_unlock_mtx, +}; + +static void genpd_lock_spin(struct generic_pm_domain *genpd) + __acquires(&genpd->slock) +{ + unsigned long flags; + + spin_lock_irqsave(&genpd->slock, flags); + genpd->lock_flags = flags; +} + +static void genpd_lock_nested_spin(struct generic_pm_domain *genpd, + int depth) + __acquires(&genpd->slock) +{ + unsigned long flags; + + spin_lock_irqsave_nested(&genpd->slock, flags, depth); + genpd->lock_flags = flags; +} + +static int genpd_lock_interruptible_spin(struct generic_pm_domain *genpd) + __acquires(&genpd->slock) +{ + unsigned long flags; + + spin_lock_irqsave(&genpd->slock, flags); + genpd->lock_flags = flags; + return 0; +} + +static void genpd_unlock_spin(struct generic_pm_domain *genpd) + __releases(&genpd->slock) +{ + spin_unlock_irqrestore(&genpd->slock, genpd->lock_flags); +} + +static const struct genpd_lock_ops genpd_spin_ops = { + .lock = genpd_lock_spin, + .lock_nested = genpd_lock_nested_spin, + .lock_interruptible = genpd_lock_interruptible_spin, + .unlock = genpd_unlock_spin, +}; + +#define genpd_lock(p) p->lock_ops->lock(p) +#define genpd_lock_nested(p, d) p->lock_ops->lock_nested(p, d) +#define genpd_lock_interruptible(p) p->lock_ops->lock_interruptible(p) +#define genpd_unlock(p) p->lock_ops->unlock(p) + +#define genpd_is_irq_safe(genpd) (genpd->flags & GENPD_FLAG_IRQ_SAFE) + +static inline bool irq_safe_dev_in_no_sleep_domain(struct device *dev, + struct generic_pm_domain *genpd) +{ + bool ret; + + ret = pm_runtime_is_irq_safe(dev) && !genpd_is_irq_safe(genpd); + + /* Warn once for each IRQ safe dev in no sleep domain */ + if (ret) + dev_warn_once(dev, "PM domain %s will not be powered off\n", + genpd->name); + + return ret; +} + /* * Get the generic PM domain for a particular struct device. * This validates the struct device pointer, the PM domain pointer, @@ -200,9 +299,9 @@ static int genpd_poweron(struct generic_pm_domain *genpd, unsigned int depth) genpd_sd_counter_inc(master); - mutex_lock_nested(&master->lock, depth + 1); + genpd_lock_nested(master, depth + 1); ret = genpd_poweron(master, depth + 1); - mutex_unlock(&master->lock); + genpd_unlock(master); if (ret) { genpd_sd_counter_dec(master); @@ -255,9 +354,9 @@ static int genpd_dev_pm_qos_notifier(struct notifier_block *nb, spin_unlock_irq(&dev->power.lock); if (!IS_ERR(genpd)) { - mutex_lock(&genpd->lock); + genpd_lock(genpd); genpd->max_off_time_changed = true; - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); } dev = dev->parent; @@ -303,7 +402,12 @@ static int genpd_poweroff(struct generic_pm_domain *genpd, bool is_async) if (stat > PM_QOS_FLAGS_NONE) return -EBUSY; - if (!pm_runtime_suspended(pdd->dev) || pdd->dev->power.irq_safe) + /* + * Do not allow PM domain to be powered off, when an IRQ safe + * device is part of a non-IRQ safe domain. + */ + if (!pm_runtime_suspended(pdd->dev) || + irq_safe_dev_in_no_sleep_domain(pdd->dev, genpd)) not_suspended++; } @@ -354,9 +458,9 @@ static void genpd_power_off_work_fn(struct work_struct *work) genpd = container_of(work, struct generic_pm_domain, power_off_work); - mutex_lock(&genpd->lock); + genpd_lock(genpd); genpd_poweroff(genpd, true); - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); } /** @@ -466,15 +570,15 @@ static int genpd_runtime_suspend(struct device *dev) } /* - * If power.irq_safe is set, this routine will be run with interrupts - * off, so it can't use mutexes. + * If power.irq_safe is set, this routine may be run with + * IRQs disabled, so suspend only if the PM domain also is irq_safe. */ - if (dev->power.irq_safe) + if (irq_safe_dev_in_no_sleep_domain(dev, genpd)) return 0; - mutex_lock(&genpd->lock); + genpd_lock(genpd); genpd_poweroff(genpd, false); - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); return 0; } @@ -503,15 +607,18 @@ static int genpd_runtime_resume(struct device *dev) if (IS_ERR(genpd)) return -EINVAL; - /* If power.irq_safe, the PM domain is never powered off. */ - if (dev->power.irq_safe) { + /* + * As we don't power off a non IRQ safe domain, which holds + * an IRQ safe device, we don't need to restore power to it. + */ + if (irq_safe_dev_in_no_sleep_domain(dev, genpd)) { timed = false; goto out; } - mutex_lock(&genpd->lock); + genpd_lock(genpd); ret = genpd_poweron(genpd, 0); - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); if (ret) return ret; @@ -546,10 +653,11 @@ static int genpd_runtime_resume(struct device *dev) err_stop: genpd_stop_dev(genpd, dev); err_poweroff: - if (!dev->power.irq_safe) { - mutex_lock(&genpd->lock); + if (!pm_runtime_is_irq_safe(dev) || + (pm_runtime_is_irq_safe(dev) && genpd_is_irq_safe(genpd))) { + genpd_lock(genpd); genpd_poweroff(genpd, 0); - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); } return ret; @@ -732,20 +840,20 @@ static int pm_genpd_prepare(struct device *dev) if (resume_needed(dev, genpd)) pm_runtime_resume(dev); - mutex_lock(&genpd->lock); + genpd_lock(genpd); if (genpd->prepared_count++ == 0) genpd->suspended_count = 0; - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); ret = pm_generic_prepare(dev); if (ret) { - mutex_lock(&genpd->lock); + genpd_lock(genpd); genpd->prepared_count--; - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); } return ret; @@ -936,13 +1044,13 @@ static void pm_genpd_complete(struct device *dev) pm_generic_complete(dev); - mutex_lock(&genpd->lock); + genpd_lock(genpd); genpd->prepared_count--; if (!genpd->prepared_count) genpd_queue_power_off_work(genpd); - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); } /** @@ -1071,7 +1179,7 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, if (IS_ERR(gpd_data)) return PTR_ERR(gpd_data); - mutex_lock(&genpd->lock); + genpd_lock(genpd); if (genpd->prepared_count > 0) { ret = -EAGAIN; @@ -1088,7 +1196,7 @@ static int genpd_add_device(struct generic_pm_domain *genpd, struct device *dev, list_add_tail(&gpd_data->base.list_node, &genpd->dev_list); out: - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); if (ret) genpd_free_dev_data(dev, gpd_data); @@ -1130,7 +1238,7 @@ static int genpd_remove_device(struct generic_pm_domain *genpd, gpd_data = to_gpd_data(pdd); dev_pm_qos_remove_notifier(dev, &gpd_data->nb); - mutex_lock(&genpd->lock); + genpd_lock(genpd); if (genpd->prepared_count > 0) { ret = -EAGAIN; @@ -1145,14 +1253,14 @@ static int genpd_remove_device(struct generic_pm_domain *genpd, list_del_init(&pdd->list_node); - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); genpd_free_dev_data(dev, gpd_data); return 0; out: - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); dev_pm_qos_add_notifier(dev, &gpd_data->nb); return ret; @@ -1183,12 +1291,23 @@ static int genpd_add_subdomain(struct generic_pm_domain *genpd, || genpd == subdomain) return -EINVAL; + /* + * If the domain can be powered on/off in an IRQ safe + * context, ensure that the subdomain can also be + * powered on/off in that context. + */ + if (!genpd_is_irq_safe(genpd) && genpd_is_irq_safe(subdomain)) { + WARN(1, "Parent %s of subdomain %s must be IRQ safe\n", + genpd->name, subdomain->name); + return -EINVAL; + } + link = kzalloc(sizeof(*link), GFP_KERNEL); if (!link) return -ENOMEM; - mutex_lock(&subdomain->lock); - mutex_lock_nested(&genpd->lock, SINGLE_DEPTH_NESTING); + genpd_lock(subdomain); + genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING); if (genpd->status == GPD_STATE_POWER_OFF && subdomain->status != GPD_STATE_POWER_OFF) { @@ -1211,8 +1330,8 @@ static int genpd_add_subdomain(struct generic_pm_domain *genpd, genpd_sd_counter_inc(genpd); out: - mutex_unlock(&genpd->lock); - mutex_unlock(&subdomain->lock); + genpd_unlock(genpd); + genpd_unlock(subdomain); if (ret) kfree(link); return ret; @@ -1250,8 +1369,8 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, if (IS_ERR_OR_NULL(genpd) || IS_ERR_OR_NULL(subdomain)) return -EINVAL; - mutex_lock(&subdomain->lock); - mutex_lock_nested(&genpd->lock, SINGLE_DEPTH_NESTING); + genpd_lock(subdomain); + genpd_lock_nested(genpd, SINGLE_DEPTH_NESTING); if (!list_empty(&subdomain->master_links) || subdomain->device_count) { pr_warn("%s: unable to remove subdomain %s\n", genpd->name, @@ -1275,13 +1394,39 @@ int pm_genpd_remove_subdomain(struct generic_pm_domain *genpd, } out: - mutex_unlock(&genpd->lock); - mutex_unlock(&subdomain->lock); + genpd_unlock(genpd); + genpd_unlock(subdomain); return ret; } EXPORT_SYMBOL_GPL(pm_genpd_remove_subdomain); +static int genpd_set_default_power_state(struct generic_pm_domain *genpd) +{ + struct genpd_power_state *state; + + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + genpd->states = state; + genpd->state_count = 1; + genpd->free = state; + + return 0; +} + +static void genpd_lock_init(struct generic_pm_domain *genpd) +{ + if (genpd->flags & GENPD_FLAG_IRQ_SAFE) { + spin_lock_init(&genpd->slock); + genpd->lock_ops = &genpd_spin_ops; + } else { + mutex_init(&genpd->mlock); + genpd->lock_ops = &genpd_mtx_ops; + } +} + /** * pm_genpd_init - Initialize a generic I/O PM domain object. * @genpd: PM domain object to initialize. @@ -1293,13 +1438,15 @@ EXPORT_SYMBOL_GPL(pm_genpd_remove_subdomain); int pm_genpd_init(struct generic_pm_domain *genpd, struct dev_power_governor *gov, bool is_off) { + int ret; + if (IS_ERR_OR_NULL(genpd)) return -EINVAL; INIT_LIST_HEAD(&genpd->master_links); INIT_LIST_HEAD(&genpd->slave_links); INIT_LIST_HEAD(&genpd->dev_list); - mutex_init(&genpd->lock); + genpd_lock_init(genpd); genpd->gov = gov; INIT_WORK(&genpd->power_off_work, genpd_power_off_work_fn); atomic_set(&genpd->sd_count, 0); @@ -1325,19 +1472,12 @@ int pm_genpd_init(struct generic_pm_domain *genpd, genpd->dev_ops.start = pm_clk_resume; } - if (genpd->state_idx >= GENPD_MAX_NUM_STATES) { - pr_warn("Initial state index out of bounds.\n"); - genpd->state_idx = GENPD_MAX_NUM_STATES - 1; - } - - if (genpd->state_count > GENPD_MAX_NUM_STATES) { - pr_warn("Limiting states to %d\n", GENPD_MAX_NUM_STATES); - genpd->state_count = GENPD_MAX_NUM_STATES; - } - /* Use only one "off" state if there were no states declared */ - if (genpd->state_count == 0) - genpd->state_count = 1; + if (genpd->state_count == 0) { + ret = genpd_set_default_power_state(genpd); + if (ret) + return ret; + } mutex_lock(&gpd_list_lock); list_add(&genpd->gpd_list_node, &gpd_list); @@ -1354,16 +1494,16 @@ static int genpd_remove(struct generic_pm_domain *genpd) if (IS_ERR_OR_NULL(genpd)) return -EINVAL; - mutex_lock(&genpd->lock); + genpd_lock(genpd); if (genpd->has_provider) { - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); pr_err("Provider present, unable to remove %s\n", genpd->name); return -EBUSY; } if (!list_empty(&genpd->master_links) || genpd->device_count) { - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); pr_err("%s: unable to remove %s\n", __func__, genpd->name); return -EBUSY; } @@ -1375,8 +1515,9 @@ static int genpd_remove(struct generic_pm_domain *genpd) } list_del(&genpd->gpd_list_node); - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); cancel_work_sync(&genpd->power_off_work); + kfree(genpd->free); pr_debug("%s: removed %s\n", __func__, genpd->name); return 0; @@ -1890,21 +2031,117 @@ int genpd_dev_pm_attach(struct device *dev) mutex_unlock(&gpd_list_lock); if (ret < 0) { - dev_err(dev, "failed to add to PM domain %s: %d", - pd->name, ret); + if (ret != -EPROBE_DEFER) + dev_err(dev, "failed to add to PM domain %s: %d", + pd->name, ret); goto out; } dev->pm_domain->detach = genpd_dev_pm_detach; dev->pm_domain->sync = genpd_dev_pm_sync; - mutex_lock(&pd->lock); + genpd_lock(pd); ret = genpd_poweron(pd, 0); - mutex_unlock(&pd->lock); + genpd_unlock(pd); out: return ret ? -EPROBE_DEFER : 0; } EXPORT_SYMBOL_GPL(genpd_dev_pm_attach); + +static const struct of_device_id idle_state_match[] = { + { .compatible = "domain-idle-state", }, + { } +}; + +static int genpd_parse_state(struct genpd_power_state *genpd_state, + struct device_node *state_node) +{ + int err; + u32 residency; + u32 entry_latency, exit_latency; + const struct of_device_id *match_id; + + match_id = of_match_node(idle_state_match, state_node); + if (!match_id) + return -EINVAL; + + err = of_property_read_u32(state_node, "entry-latency-us", + &entry_latency); + if (err) { + pr_debug(" * %s missing entry-latency-us property\n", + state_node->full_name); + return -EINVAL; + } + + err = of_property_read_u32(state_node, "exit-latency-us", + &exit_latency); + if (err) { + pr_debug(" * %s missing exit-latency-us property\n", + state_node->full_name); + return -EINVAL; + } + + err = of_property_read_u32(state_node, "min-residency-us", &residency); + if (!err) + genpd_state->residency_ns = 1000 * residency; + + genpd_state->power_on_latency_ns = 1000 * exit_latency; + genpd_state->power_off_latency_ns = 1000 * entry_latency; + genpd_state->fwnode = &state_node->fwnode; + + return 0; +} + +/** + * of_genpd_parse_idle_states: Return array of idle states for the genpd. + * + * @dn: The genpd device node + * @states: The pointer to which the state array will be saved. + * @n: The count of elements in the array returned from this function. + * + * Returns the device states parsed from the OF node. The memory for the states + * is allocated by this function and is the responsibility of the caller to + * free the memory after use. + */ +int of_genpd_parse_idle_states(struct device_node *dn, + struct genpd_power_state **states, int *n) +{ + struct genpd_power_state *st; + struct device_node *np; + int i = 0; + int err, ret; + int count; + struct of_phandle_iterator it; + + count = of_count_phandle_with_args(dn, "domain-idle-states", NULL); + if (count <= 0) + return -EINVAL; + + st = kcalloc(count, sizeof(*st), GFP_KERNEL); + if (!st) + return -ENOMEM; + + /* Loop over the phandles until all the requested entry is found */ + of_for_each_phandle(&it, err, dn, "domain-idle-states", NULL, 0) { + np = it.node; + ret = genpd_parse_state(&st[i++], np); + if (ret) { + pr_err + ("Parsing idle state node %s failed with err %d\n", + np->full_name, ret); + of_node_put(np); + kfree(st); + return ret; + } + } + + *n = count; + *states = st; + + return 0; +} +EXPORT_SYMBOL_GPL(of_genpd_parse_idle_states); + #endif /* CONFIG_PM_GENERIC_DOMAINS_OF */ @@ -1958,7 +2195,7 @@ static int pm_genpd_summary_one(struct seq_file *s, char state[16]; int ret; - ret = mutex_lock_interruptible(&genpd->lock); + ret = genpd_lock_interruptible(genpd); if (ret) return -ERESTARTSYS; @@ -1984,7 +2221,9 @@ static int pm_genpd_summary_one(struct seq_file *s, } list_for_each_entry(pm_data, &genpd->dev_list, list_node) { - kobj_path = kobject_get_path(&pm_data->dev->kobj, GFP_KERNEL); + kobj_path = kobject_get_path(&pm_data->dev->kobj, + genpd_is_irq_safe(genpd) ? + GFP_ATOMIC : GFP_KERNEL); if (kobj_path == NULL) continue; @@ -1995,7 +2234,7 @@ static int pm_genpd_summary_one(struct seq_file *s, seq_puts(s, "\n"); exit: - mutex_unlock(&genpd->lock); + genpd_unlock(genpd); return 0; } diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 2932a5bd892f..eb474c882ebe 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -1460,10 +1460,10 @@ static int __device_suspend(struct device *dev, pm_message_t state, bool async) dpm_watchdog_clear(&wd); Complete: - complete_all(&dev->power.completion); if (error) async_error = error; + complete_all(&dev->power.completion); TRACE_SUSPEND(error); return error; } diff --git a/drivers/base/power/opp/core.c b/drivers/base/power/opp/core.c index 4c7c6da7a989..35ff06283738 100644 --- a/drivers/base/power/opp/core.c +++ b/drivers/base/power/opp/core.c @@ -93,6 +93,8 @@ struct opp_table *_find_opp_table(struct device *dev) * Return: voltage in micro volt corresponding to the opp, else * return 0 * + * This is useful only for devices with single power supply. + * * Locking: This function must be called under rcu_read_lock(). opp is a rcu * protected pointer. This means that opp which could have been fetched by * opp_find_freq_{exact,ceil,floor} functions is valid as long as we are @@ -112,7 +114,7 @@ unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp) if (IS_ERR_OR_NULL(tmp_opp)) pr_err("%s: Invalid parameters\n", __func__); else - v = tmp_opp->u_volt; + v = tmp_opp->supplies[0].u_volt; return v; } @@ -210,6 +212,24 @@ unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev) } EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency); +static int _get_regulator_count(struct device *dev) +{ + struct opp_table *opp_table; + int count; + + rcu_read_lock(); + + opp_table = _find_opp_table(dev); + if (!IS_ERR(opp_table)) + count = opp_table->regulator_count; + else + count = 0; + + rcu_read_unlock(); + + return count; +} + /** * dev_pm_opp_get_max_volt_latency() - Get max voltage latency in nanoseconds * @dev: device for which we do this operation @@ -222,34 +242,51 @@ unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev) { struct opp_table *opp_table; struct dev_pm_opp *opp; - struct regulator *reg; + struct regulator *reg, **regulators; unsigned long latency_ns = 0; - unsigned long min_uV = ~0, max_uV = 0; - int ret; + int ret, i, count; + struct { + unsigned long min; + unsigned long max; + } *uV; + + count = _get_regulator_count(dev); + + /* Regulator may not be required for the device */ + if (!count) + return 0; + + regulators = kmalloc_array(count, sizeof(*regulators), GFP_KERNEL); + if (!regulators) + return 0; + + uV = kmalloc_array(count, sizeof(*uV), GFP_KERNEL); + if (!uV) + goto free_regulators; rcu_read_lock(); opp_table = _find_opp_table(dev); if (IS_ERR(opp_table)) { rcu_read_unlock(); - return 0; + goto free_uV; } - reg = opp_table->regulator; - if (IS_ERR(reg)) { - /* Regulator may not be required for device */ - rcu_read_unlock(); - return 0; - } + memcpy(regulators, opp_table->regulators, count * sizeof(*regulators)); - list_for_each_entry_rcu(opp, &opp_table->opp_list, node) { - if (!opp->available) - continue; + for (i = 0; i < count; i++) { + uV[i].min = ~0; + uV[i].max = 0; + + list_for_each_entry_rcu(opp, &opp_table->opp_list, node) { + if (!opp->available) + continue; - if (opp->u_volt_min < min_uV) - min_uV = opp->u_volt_min; - if (opp->u_volt_max > max_uV) - max_uV = opp->u_volt_max; + if (opp->supplies[i].u_volt_min < uV[i].min) + uV[i].min = opp->supplies[i].u_volt_min; + if (opp->supplies[i].u_volt_max > uV[i].max) + uV[i].max = opp->supplies[i].u_volt_max; + } } rcu_read_unlock(); @@ -258,9 +295,16 @@ unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev) * The caller needs to ensure that opp_table (and hence the regulator) * isn't freed, while we are executing this routine. */ - ret = regulator_set_voltage_time(reg, min_uV, max_uV); - if (ret > 0) - latency_ns = ret * 1000; + for (i = 0; reg = regulators[i], i < count; i++) { + ret = regulator_set_voltage_time(reg, uV[i].min, uV[i].max); + if (ret > 0) + latency_ns += ret * 1000; + } + +free_uV: + kfree(uV); +free_regulators: + kfree(regulators); return latency_ns; } @@ -542,8 +586,7 @@ unlock: } static int _set_opp_voltage(struct device *dev, struct regulator *reg, - unsigned long u_volt, unsigned long u_volt_min, - unsigned long u_volt_max) + struct dev_pm_opp_supply *supply) { int ret; @@ -554,14 +597,78 @@ static int _set_opp_voltage(struct device *dev, struct regulator *reg, return 0; } - dev_dbg(dev, "%s: voltages (mV): %lu %lu %lu\n", __func__, u_volt_min, - u_volt, u_volt_max); + dev_dbg(dev, "%s: voltages (mV): %lu %lu %lu\n", __func__, + supply->u_volt_min, supply->u_volt, supply->u_volt_max); - ret = regulator_set_voltage_triplet(reg, u_volt_min, u_volt, - u_volt_max); + ret = regulator_set_voltage_triplet(reg, supply->u_volt_min, + supply->u_volt, supply->u_volt_max); if (ret) dev_err(dev, "%s: failed to set voltage (%lu %lu %lu mV): %d\n", - __func__, u_volt_min, u_volt, u_volt_max, ret); + __func__, supply->u_volt_min, supply->u_volt, + supply->u_volt_max, ret); + + return ret; +} + +static inline int +_generic_set_opp_clk_only(struct device *dev, struct clk *clk, + unsigned long old_freq, unsigned long freq) +{ + int ret; + + ret = clk_set_rate(clk, freq); + if (ret) { + dev_err(dev, "%s: failed to set clock rate: %d\n", __func__, + ret); + } + + return ret; +} + +static int _generic_set_opp(struct dev_pm_set_opp_data *data) +{ + struct dev_pm_opp_supply *old_supply = data->old_opp.supplies; + struct dev_pm_opp_supply *new_supply = data->new_opp.supplies; + unsigned long old_freq = data->old_opp.rate, freq = data->new_opp.rate; + struct regulator *reg = data->regulators[0]; + struct device *dev= data->dev; + int ret; + + /* This function only supports single regulator per device */ + if (WARN_ON(data->regulator_count > 1)) { + dev_err(dev, "multiple regulators are not supported\n"); + return -EINVAL; + } + + /* Scaling up? Scale voltage before frequency */ + if (freq > old_freq) { + ret = _set_opp_voltage(dev, reg, new_supply); + if (ret) + goto restore_voltage; + } + + /* Change frequency */ + ret = _generic_set_opp_clk_only(dev, data->clk, old_freq, freq); + if (ret) + goto restore_voltage; + + /* Scaling down? Scale voltage after frequency */ + if (freq < old_freq) { + ret = _set_opp_voltage(dev, reg, new_supply); + if (ret) + goto restore_freq; + } + + return 0; + +restore_freq: + if (_generic_set_opp_clk_only(dev, data->clk, freq, old_freq)) + dev_err(dev, "%s: failed to restore old-freq (%lu Hz)\n", + __func__, old_freq); +restore_voltage: + /* This shouldn't harm even if the voltages weren't updated earlier */ + if (old_supply->u_volt) + _set_opp_voltage(dev, reg, old_supply); return ret; } @@ -579,12 +686,13 @@ static int _set_opp_voltage(struct device *dev, struct regulator *reg, int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) { struct opp_table *opp_table; + unsigned long freq, old_freq; + int (*set_opp)(struct dev_pm_set_opp_data *data); struct dev_pm_opp *old_opp, *opp; - struct regulator *reg; + struct regulator **regulators; + struct dev_pm_set_opp_data *data; struct clk *clk; - unsigned long freq, old_freq; - unsigned long u_volt, u_volt_min, u_volt_max; - int ret; + int ret, size; if (unlikely(!target_freq)) { dev_err(dev, "%s: Invalid target frequency %lu\n", __func__, @@ -633,55 +741,41 @@ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) return ret; } - u_volt = opp->u_volt; - u_volt_min = opp->u_volt_min; - u_volt_max = opp->u_volt_max; + dev_dbg(dev, "%s: switching OPP: %lu Hz --> %lu Hz\n", __func__, + old_freq, freq); - reg = opp_table->regulator; + regulators = opp_table->regulators; - rcu_read_unlock(); - - /* Scaling up? Scale voltage before frequency */ - if (freq > old_freq) { - ret = _set_opp_voltage(dev, reg, u_volt, u_volt_min, - u_volt_max); - if (ret) - goto restore_voltage; - } - - /* Change frequency */ - - dev_dbg(dev, "%s: switching OPP: %lu Hz --> %lu Hz\n", - __func__, old_freq, freq); - - ret = clk_set_rate(clk, freq); - if (ret) { - dev_err(dev, "%s: failed to set clock rate: %d\n", __func__, - ret); - goto restore_voltage; + /* Only frequency scaling */ + if (!regulators) { + rcu_read_unlock(); + return _generic_set_opp_clk_only(dev, clk, old_freq, freq); } - /* Scaling down? Scale voltage after frequency */ - if (freq < old_freq) { - ret = _set_opp_voltage(dev, reg, u_volt, u_volt_min, - u_volt_max); - if (ret) - goto restore_freq; - } + if (opp_table->set_opp) + set_opp = opp_table->set_opp; + else + set_opp = _generic_set_opp; + + data = opp_table->set_opp_data; + data->regulators = regulators; + data->regulator_count = opp_table->regulator_count; + data->clk = clk; + data->dev = dev; + + data->old_opp.rate = old_freq; + size = sizeof(*opp->supplies) * opp_table->regulator_count; + if (IS_ERR(old_opp)) + memset(data->old_opp.supplies, 0, size); + else + memcpy(data->old_opp.supplies, old_opp->supplies, size); - return 0; + data->new_opp.rate = freq; + memcpy(data->new_opp.supplies, opp->supplies, size); -restore_freq: - if (clk_set_rate(clk, old_freq)) - dev_err(dev, "%s: failed to restore old-freq (%lu Hz)\n", - __func__, old_freq); -restore_voltage: - /* This shouldn't harm even if the voltages weren't updated earlier */ - if (!IS_ERR(old_opp)) - _set_opp_voltage(dev, reg, old_opp->u_volt, - old_opp->u_volt_min, old_opp->u_volt_max); + rcu_read_unlock(); - return ret; + return set_opp(data); } EXPORT_SYMBOL_GPL(dev_pm_opp_set_rate); @@ -764,9 +858,6 @@ static struct opp_table *_add_opp_table(struct device *dev) _of_init_opp_table(opp_table, dev); - /* Set regulator to a non-NULL error value */ - opp_table->regulator = ERR_PTR(-ENXIO); - /* Find clk for the device */ opp_table->clk = clk_get(dev, NULL); if (IS_ERR(opp_table->clk)) { @@ -815,7 +906,10 @@ static void _remove_opp_table(struct opp_table *opp_table) if (opp_table->prop_name) return; - if (!IS_ERR(opp_table->regulator)) + if (opp_table->regulators) + return; + + if (opp_table->set_opp) return; /* Release clk */ @@ -924,34 +1018,50 @@ struct dev_pm_opp *_allocate_opp(struct device *dev, struct opp_table **opp_table) { struct dev_pm_opp *opp; + int count, supply_size; + struct opp_table *table; - /* allocate new OPP node */ - opp = kzalloc(sizeof(*opp), GFP_KERNEL); - if (!opp) + table = _add_opp_table(dev); + if (!table) return NULL; - INIT_LIST_HEAD(&opp->node); + /* Allocate space for at least one supply */ + count = table->regulator_count ? table->regulator_count : 1; + supply_size = sizeof(*opp->supplies) * count; - *opp_table = _add_opp_table(dev); - if (!*opp_table) { - kfree(opp); + /* allocate new OPP node and supplies structures */ + opp = kzalloc(sizeof(*opp) + supply_size, GFP_KERNEL); + if (!opp) { + kfree(table); return NULL; } + /* Put the supplies at the end of the OPP structure as an empty array */ + opp->supplies = (struct dev_pm_opp_supply *)(opp + 1); + INIT_LIST_HEAD(&opp->node); + + *opp_table = table; + return opp; } static bool _opp_supported_by_regulators(struct dev_pm_opp *opp, struct opp_table *opp_table) { - struct regulator *reg = opp_table->regulator; - - if (!IS_ERR(reg) && - !regulator_is_supported_voltage(reg, opp->u_volt_min, - opp->u_volt_max)) { - pr_warn("%s: OPP minuV: %lu maxuV: %lu, not supported by regulator\n", - __func__, opp->u_volt_min, opp->u_volt_max); - return false; + struct regulator *reg; + int i; + + for (i = 0; i < opp_table->regulator_count; i++) { + reg = opp_table->regulators[i]; + + if (!regulator_is_supported_voltage(reg, + opp->supplies[i].u_volt_min, + opp->supplies[i].u_volt_max)) { + pr_warn("%s: OPP minuV: %lu maxuV: %lu, not supported by regulator\n", + __func__, opp->supplies[i].u_volt_min, + opp->supplies[i].u_volt_max); + return false; + } } return true; @@ -983,11 +1093,13 @@ int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, /* Duplicate OPPs */ dev_warn(dev, "%s: duplicate OPPs detected. Existing: freq: %lu, volt: %lu, enabled: %d. New: freq: %lu, volt: %lu, enabled: %d\n", - __func__, opp->rate, opp->u_volt, opp->available, - new_opp->rate, new_opp->u_volt, new_opp->available); + __func__, opp->rate, opp->supplies[0].u_volt, + opp->available, new_opp->rate, + new_opp->supplies[0].u_volt, new_opp->available); - return opp->available && new_opp->u_volt == opp->u_volt ? - 0 : -EEXIST; + /* Should we compare voltages for all regulators here ? */ + return opp->available && + new_opp->supplies[0].u_volt == opp->supplies[0].u_volt ? 0 : -EEXIST; } new_opp->opp_table = opp_table; @@ -1054,9 +1166,9 @@ int _opp_add_v1(struct device *dev, unsigned long freq, long u_volt, /* populate the opp table */ new_opp->rate = freq; tol = u_volt * opp_table->voltage_tolerance_v1 / 100; - new_opp->u_volt = u_volt; - new_opp->u_volt_min = u_volt - tol; - new_opp->u_volt_max = u_volt + tol; + new_opp->supplies[0].u_volt = u_volt; + new_opp->supplies[0].u_volt_min = u_volt - tol; + new_opp->supplies[0].u_volt_max = u_volt + tol; new_opp->available = true; new_opp->dynamic = dynamic; @@ -1300,13 +1412,47 @@ unlock: } EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name); +static int _allocate_set_opp_data(struct opp_table *opp_table) +{ + struct dev_pm_set_opp_data *data; + int len, count = opp_table->regulator_count; + + if (WARN_ON(!count)) + return -EINVAL; + + /* space for set_opp_data */ + len = sizeof(*data); + + /* space for old_opp.supplies and new_opp.supplies */ + len += 2 * sizeof(struct dev_pm_opp_supply) * count; + + data = kzalloc(len, GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->old_opp.supplies = (void *)(data + 1); + data->new_opp.supplies = data->old_opp.supplies + count; + + opp_table->set_opp_data = data; + + return 0; +} + +static void _free_set_opp_data(struct opp_table *opp_table) +{ + kfree(opp_table->set_opp_data); + opp_table->set_opp_data = NULL; +} + /** - * dev_pm_opp_set_regulator() - Set regulator name for the device + * dev_pm_opp_set_regulators() - Set regulator names for the device * @dev: Device for which regulator name is being set. - * @name: Name of the regulator. + * @names: Array of pointers to the names of the regulator. + * @count: Number of regulators. * * In order to support OPP switching, OPP layer needs to know the name of the - * device's regulator, as the core would be required to switch voltages as well. + * device's regulators, as the core would be required to switch voltages as + * well. * * This must be called before any OPPs are initialized for the device. * @@ -1316,11 +1462,13 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_put_prop_name); * that this function is *NOT* called under RCU protection or in contexts where * mutex cannot be locked. */ -int dev_pm_opp_set_regulator(struct device *dev, const char *name) +struct opp_table *dev_pm_opp_set_regulators(struct device *dev, + const char * const names[], + unsigned int count) { struct opp_table *opp_table; struct regulator *reg; - int ret; + int ret, i; mutex_lock(&opp_table_lock); @@ -1336,22 +1484,146 @@ int dev_pm_opp_set_regulator(struct device *dev, const char *name) goto err; } - /* Already have a regulator set */ - if (WARN_ON(!IS_ERR(opp_table->regulator))) { + /* Already have regulators set */ + if (opp_table->regulators) { ret = -EBUSY; goto err; } - /* Allocate the regulator */ - reg = regulator_get_optional(dev, name); - if (IS_ERR(reg)) { - ret = PTR_ERR(reg); - if (ret != -EPROBE_DEFER) - dev_err(dev, "%s: no regulator (%s) found: %d\n", - __func__, name, ret); + + opp_table->regulators = kmalloc_array(count, + sizeof(*opp_table->regulators), + GFP_KERNEL); + if (!opp_table->regulators) { + ret = -ENOMEM; + goto err; + } + + for (i = 0; i < count; i++) { + reg = regulator_get_optional(dev, names[i]); + if (IS_ERR(reg)) { + ret = PTR_ERR(reg); + if (ret != -EPROBE_DEFER) + dev_err(dev, "%s: no regulator (%s) found: %d\n", + __func__, names[i], ret); + goto free_regulators; + } + + opp_table->regulators[i] = reg; + } + + opp_table->regulator_count = count; + + /* Allocate block only once to pass to set_opp() routines */ + ret = _allocate_set_opp_data(opp_table); + if (ret) + goto free_regulators; + + mutex_unlock(&opp_table_lock); + return opp_table; + +free_regulators: + while (i != 0) + regulator_put(opp_table->regulators[--i]); + + kfree(opp_table->regulators); + opp_table->regulators = NULL; + opp_table->regulator_count = 0; +err: + _remove_opp_table(opp_table); +unlock: + mutex_unlock(&opp_table_lock); + + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulators); + +/** + * dev_pm_opp_put_regulators() - Releases resources blocked for regulator + * @opp_table: OPP table returned from dev_pm_opp_set_regulators(). + * + * Locking: The internal opp_table and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +void dev_pm_opp_put_regulators(struct opp_table *opp_table) +{ + int i; + + mutex_lock(&opp_table_lock); + + if (!opp_table->regulators) { + pr_err("%s: Doesn't have regulators set\n", __func__); + goto unlock; + } + + /* Make sure there are no concurrent readers while updating opp_table */ + WARN_ON(!list_empty(&opp_table->opp_list)); + + for (i = opp_table->regulator_count - 1; i >= 0; i--) + regulator_put(opp_table->regulators[i]); + + _free_set_opp_data(opp_table); + + kfree(opp_table->regulators); + opp_table->regulators = NULL; + opp_table->regulator_count = 0; + + /* Try freeing opp_table if this was the last blocking resource */ + _remove_opp_table(opp_table); + +unlock: + mutex_unlock(&opp_table_lock); +} +EXPORT_SYMBOL_GPL(dev_pm_opp_put_regulators); + +/** + * dev_pm_opp_register_set_opp_helper() - Register custom set OPP helper + * @dev: Device for which the helper is getting registered. + * @set_opp: Custom set OPP helper. + * + * This is useful to support complex platforms (like platforms with multiple + * regulators per device), instead of the generic OPP set rate helper. + * + * This must be called before any OPPs are initialized for the device. + * + * Locking: The internal opp_table and opp structures are RCU protected. + * Hence this function internally uses RCU updater strategy with mutex locks + * to keep the integrity of the internal data structures. Callers should ensure + * that this function is *NOT* called under RCU protection or in contexts where + * mutex cannot be locked. + */ +int dev_pm_opp_register_set_opp_helper(struct device *dev, + int (*set_opp)(struct dev_pm_set_opp_data *data)) +{ + struct opp_table *opp_table; + int ret; + + if (!set_opp) + return -EINVAL; + + mutex_lock(&opp_table_lock); + + opp_table = _add_opp_table(dev); + if (!opp_table) { + ret = -ENOMEM; + goto unlock; + } + + /* This should be called before OPPs are initialized */ + if (WARN_ON(!list_empty(&opp_table->opp_list))) { + ret = -EBUSY; goto err; } - opp_table->regulator = reg; + /* Already have custom set_opp helper */ + if (WARN_ON(opp_table->set_opp)) { + ret = -EBUSY; + goto err; + } + + opp_table->set_opp = set_opp; mutex_unlock(&opp_table_lock); return 0; @@ -1363,11 +1635,12 @@ unlock: return ret; } -EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulator); +EXPORT_SYMBOL_GPL(dev_pm_opp_register_set_opp_helper); /** - * dev_pm_opp_put_regulator() - Releases resources blocked for regulator - * @dev: Device for which regulator was set. + * dev_pm_opp_register_put_opp_helper() - Releases resources blocked for + * set_opp helper + * @dev: Device for which custom set_opp helper has to be cleared. * * Locking: The internal opp_table and opp structures are RCU protected. * Hence this function internally uses RCU updater strategy with mutex locks @@ -1375,7 +1648,7 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_set_regulator); * that this function is *NOT* called under RCU protection or in contexts where * mutex cannot be locked. */ -void dev_pm_opp_put_regulator(struct device *dev) +void dev_pm_opp_register_put_opp_helper(struct device *dev) { struct opp_table *opp_table; @@ -1389,16 +1662,16 @@ void dev_pm_opp_put_regulator(struct device *dev) goto unlock; } - if (IS_ERR(opp_table->regulator)) { - dev_err(dev, "%s: Doesn't have regulator set\n", __func__); + if (!opp_table->set_opp) { + dev_err(dev, "%s: Doesn't have custom set_opp helper set\n", + __func__); goto unlock; } /* Make sure there are no concurrent readers while updating opp_table */ WARN_ON(!list_empty(&opp_table->opp_list)); - regulator_put(opp_table->regulator); - opp_table->regulator = ERR_PTR(-ENXIO); + opp_table->set_opp = NULL; /* Try freeing opp_table if this was the last blocking resource */ _remove_opp_table(opp_table); @@ -1406,7 +1679,7 @@ void dev_pm_opp_put_regulator(struct device *dev) unlock: mutex_unlock(&opp_table_lock); } -EXPORT_SYMBOL_GPL(dev_pm_opp_put_regulator); +EXPORT_SYMBOL_GPL(dev_pm_opp_register_put_opp_helper); /** * dev_pm_opp_add() - Add an OPP table from a table definitions diff --git a/drivers/base/power/opp/debugfs.c b/drivers/base/power/opp/debugfs.c index ef1ae6b52042..95f433db4ac7 100644 --- a/drivers/base/power/opp/debugfs.c +++ b/drivers/base/power/opp/debugfs.c @@ -15,6 +15,7 @@ #include <linux/err.h> #include <linux/init.h> #include <linux/limits.h> +#include <linux/slab.h> #include "opp.h" @@ -34,6 +35,46 @@ void opp_debug_remove_one(struct dev_pm_opp *opp) debugfs_remove_recursive(opp->dentry); } +static bool opp_debug_create_supplies(struct dev_pm_opp *opp, + struct opp_table *opp_table, + struct dentry *pdentry) +{ + struct dentry *d; + int i = 0; + char *name; + + /* Always create at least supply-0 directory */ + do { + name = kasprintf(GFP_KERNEL, "supply-%d", i); + + /* Create per-opp directory */ + d = debugfs_create_dir(name, pdentry); + + kfree(name); + + if (!d) + return false; + + if (!debugfs_create_ulong("u_volt_target", S_IRUGO, d, + &opp->supplies[i].u_volt)) + return false; + + if (!debugfs_create_ulong("u_volt_min", S_IRUGO, d, + &opp->supplies[i].u_volt_min)) + return false; + + if (!debugfs_create_ulong("u_volt_max", S_IRUGO, d, + &opp->supplies[i].u_volt_max)) + return false; + + if (!debugfs_create_ulong("u_amp", S_IRUGO, d, + &opp->supplies[i].u_amp)) + return false; + } while (++i < opp_table->regulator_count); + + return true; +} + int opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table) { struct dentry *pdentry = opp_table->dentry; @@ -63,16 +104,7 @@ int opp_debug_create_one(struct dev_pm_opp *opp, struct opp_table *opp_table) if (!debugfs_create_ulong("rate_hz", S_IRUGO, d, &opp->rate)) return -ENOMEM; - if (!debugfs_create_ulong("u_volt_target", S_IRUGO, d, &opp->u_volt)) - return -ENOMEM; - - if (!debugfs_create_ulong("u_volt_min", S_IRUGO, d, &opp->u_volt_min)) - return -ENOMEM; - - if (!debugfs_create_ulong("u_volt_max", S_IRUGO, d, &opp->u_volt_max)) - return -ENOMEM; - - if (!debugfs_create_ulong("u_amp", S_IRUGO, d, &opp->u_amp)) + if (!opp_debug_create_supplies(opp, opp_table, d)) return -ENOMEM; if (!debugfs_create_ulong("clock_latency_ns", S_IRUGO, d, diff --git a/drivers/base/power/opp/of.c b/drivers/base/power/opp/of.c index 5552211e6fcd..3f7d2591b173 100644 --- a/drivers/base/power/opp/of.c +++ b/drivers/base/power/opp/of.c @@ -17,6 +17,7 @@ #include <linux/errno.h> #include <linux/device.h> #include <linux/of.h> +#include <linux/slab.h> #include <linux/export.h> #include "opp.h" @@ -101,16 +102,16 @@ static bool _opp_is_supported(struct device *dev, struct opp_table *opp_table, return true; } -/* TODO: Support multiple regulators */ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev, struct opp_table *opp_table) { - u32 microvolt[3] = {0}; - u32 val; - int count, ret; + u32 *microvolt, *microamp = NULL; + int supplies, vcount, icount, ret, i, j; struct property *prop = NULL; char name[NAME_MAX]; + supplies = opp_table->regulator_count ? opp_table->regulator_count : 1; + /* Search for "opp-microvolt-<name>" */ if (opp_table->prop_name) { snprintf(name, sizeof(name), "opp-microvolt-%s", @@ -128,34 +129,29 @@ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev, return 0; } - count = of_property_count_u32_elems(opp->np, name); - if (count < 0) { + vcount = of_property_count_u32_elems(opp->np, name); + if (vcount < 0) { dev_err(dev, "%s: Invalid %s property (%d)\n", - __func__, name, count); - return count; + __func__, name, vcount); + return vcount; } - /* There can be one or three elements here */ - if (count != 1 && count != 3) { - dev_err(dev, "%s: Invalid number of elements in %s property (%d)\n", - __func__, name, count); + /* There can be one or three elements per supply */ + if (vcount != supplies && vcount != supplies * 3) { + dev_err(dev, "%s: Invalid number of elements in %s property (%d) with supplies (%d)\n", + __func__, name, vcount, supplies); return -EINVAL; } - ret = of_property_read_u32_array(opp->np, name, microvolt, count); + microvolt = kmalloc_array(vcount, sizeof(*microvolt), GFP_KERNEL); + if (!microvolt) + return -ENOMEM; + + ret = of_property_read_u32_array(opp->np, name, microvolt, vcount); if (ret) { dev_err(dev, "%s: error parsing %s: %d\n", __func__, name, ret); - return -EINVAL; - } - - opp->u_volt = microvolt[0]; - - if (count == 1) { - opp->u_volt_min = opp->u_volt; - opp->u_volt_max = opp->u_volt; - } else { - opp->u_volt_min = microvolt[1]; - opp->u_volt_max = microvolt[2]; + ret = -EINVAL; + goto free_microvolt; } /* Search for "opp-microamp-<name>" */ @@ -172,10 +168,59 @@ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev, prop = of_find_property(opp->np, name, NULL); } - if (prop && !of_property_read_u32(opp->np, name, &val)) - opp->u_amp = val; + if (prop) { + icount = of_property_count_u32_elems(opp->np, name); + if (icount < 0) { + dev_err(dev, "%s: Invalid %s property (%d)\n", __func__, + name, icount); + ret = icount; + goto free_microvolt; + } - return 0; + if (icount != supplies) { + dev_err(dev, "%s: Invalid number of elements in %s property (%d) with supplies (%d)\n", + __func__, name, icount, supplies); + ret = -EINVAL; + goto free_microvolt; + } + + microamp = kmalloc_array(icount, sizeof(*microamp), GFP_KERNEL); + if (!microamp) { + ret = -EINVAL; + goto free_microvolt; + } + + ret = of_property_read_u32_array(opp->np, name, microamp, + icount); + if (ret) { + dev_err(dev, "%s: error parsing %s: %d\n", __func__, + name, ret); + ret = -EINVAL; + goto free_microamp; + } + } + + for (i = 0, j = 0; i < supplies; i++) { + opp->supplies[i].u_volt = microvolt[j++]; + + if (vcount == supplies) { + opp->supplies[i].u_volt_min = opp->supplies[i].u_volt; + opp->supplies[i].u_volt_max = opp->supplies[i].u_volt; + } else { + opp->supplies[i].u_volt_min = microvolt[j++]; + opp->supplies[i].u_volt_max = microvolt[j++]; + } + + if (microamp) + opp->supplies[i].u_amp = microamp[i]; + } + +free_microamp: + kfree(microamp); +free_microvolt: + kfree(microvolt); + + return ret; } /** @@ -198,7 +243,7 @@ void dev_pm_opp_of_remove_table(struct device *dev) EXPORT_SYMBOL_GPL(dev_pm_opp_of_remove_table); /* Returns opp descriptor node for a device, caller must do of_node_put() */ -struct device_node *_of_get_opp_desc_node(struct device *dev) +static struct device_node *_of_get_opp_desc_node(struct device *dev) { /* * TODO: Support for multiple OPP tables. @@ -303,9 +348,9 @@ static int _opp_add_static_v2(struct device *dev, struct device_node *np) mutex_unlock(&opp_table_lock); pr_debug("%s: turbo:%d rate:%lu uv:%lu uvmin:%lu uvmax:%lu latency:%lu\n", - __func__, new_opp->turbo, new_opp->rate, new_opp->u_volt, - new_opp->u_volt_min, new_opp->u_volt_max, - new_opp->clock_latency_ns); + __func__, new_opp->turbo, new_opp->rate, + new_opp->supplies[0].u_volt, new_opp->supplies[0].u_volt_min, + new_opp->supplies[0].u_volt_max, new_opp->clock_latency_ns); /* * Notify the changes in the availability of the operable @@ -562,7 +607,7 @@ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, /* Get OPP descriptor node */ np = _of_get_opp_desc_node(cpu_dev); if (!np) { - dev_dbg(cpu_dev, "%s: Couldn't find cpu_dev node.\n", __func__); + dev_dbg(cpu_dev, "%s: Couldn't find opp node.\n", __func__); return -ENOENT; } @@ -587,7 +632,7 @@ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, /* Get OPP descriptor node */ tmp_np = _of_get_opp_desc_node(tcpu_dev); if (!tmp_np) { - dev_err(tcpu_dev, "%s: Couldn't find tcpu_dev node.\n", + dev_err(tcpu_dev, "%s: Couldn't find opp node.\n", __func__); ret = -ENOENT; goto put_cpu_node; diff --git a/drivers/base/power/opp/opp.h b/drivers/base/power/opp/opp.h index fabd5ca1a083..af9f2b849a66 100644 --- a/drivers/base/power/opp/opp.h +++ b/drivers/base/power/opp/opp.h @@ -61,10 +61,7 @@ extern struct list_head opp_tables; * @turbo: true if turbo (boost) OPP * @suspend: true if suspend OPP * @rate: Frequency in hertz - * @u_volt: Target voltage in microvolts corresponding to this OPP - * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP - * @u_volt_max: Maximum voltage in microvolts corresponding to this OPP - * @u_amp: Maximum current drawn by the device in microamperes + * @supplies: Power supplies voltage/current values * @clock_latency_ns: Latency (in nanoseconds) of switching to this OPP's * frequency from any other OPP's frequency. * @opp_table: points back to the opp_table struct this opp belongs to @@ -83,10 +80,8 @@ struct dev_pm_opp { bool suspend; unsigned long rate; - unsigned long u_volt; - unsigned long u_volt_min; - unsigned long u_volt_max; - unsigned long u_amp; + struct dev_pm_opp_supply *supplies; + unsigned long clock_latency_ns; struct opp_table *opp_table; @@ -144,7 +139,10 @@ enum opp_table_access { * @supported_hw_count: Number of elements in supported_hw array. * @prop_name: A name to postfix to many DT properties, while parsing them. * @clk: Device's clock handle - * @regulator: Supply regulator + * @regulators: Supply regulators + * @regulator_count: Number of power supply regulators + * @set_opp: Platform specific set_opp callback + * @set_opp_data: Data to be passed to set_opp callback * @dentry: debugfs dentry pointer of the real device directory (not links). * @dentry_name: Name of the real dentry. * @@ -179,7 +177,11 @@ struct opp_table { unsigned int supported_hw_count; const char *prop_name; struct clk *clk; - struct regulator *regulator; + struct regulator **regulators; + unsigned int regulator_count; + + int (*set_opp)(struct dev_pm_set_opp_data *data); + struct dev_pm_set_opp_data *set_opp_data; #ifdef CONFIG_DEBUG_FS struct dentry *dentry; @@ -190,7 +192,6 @@ struct opp_table { /* Routines internal to opp core */ struct opp_table *_find_opp_table(struct device *dev); struct opp_device *_add_opp_dev(const struct device *dev, struct opp_table *opp_table); -struct device_node *_of_get_opp_desc_node(struct device *dev); void _dev_pm_opp_remove_table(struct device *dev, bool remove_all); struct dev_pm_opp *_allocate_opp(struct device *dev, struct opp_table **opp_table); int _opp_add(struct device *dev, struct dev_pm_opp *new_opp, struct opp_table *opp_table); diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h index 50e30e7b059d..a84332aefc2d 100644 --- a/drivers/base/power/power.h +++ b/drivers/base/power/power.h @@ -21,14 +21,22 @@ extern void pm_runtime_init(struct device *dev); extern void pm_runtime_reinit(struct device *dev); extern void pm_runtime_remove(struct device *dev); +#define WAKE_IRQ_DEDICATED_ALLOCATED BIT(0) +#define WAKE_IRQ_DEDICATED_MANAGED BIT(1) +#define WAKE_IRQ_DEDICATED_MASK (WAKE_IRQ_DEDICATED_ALLOCATED | \ + WAKE_IRQ_DEDICATED_MANAGED) + struct wake_irq { struct device *dev; + unsigned int status; int irq; - bool dedicated_irq:1; }; extern void dev_pm_arm_wake_irq(struct wake_irq *wirq); extern void dev_pm_disarm_wake_irq(struct wake_irq *wirq); +extern void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status); +extern void dev_pm_disable_wake_irq_check(struct device *dev); #ifdef CONFIG_PM_SLEEP @@ -104,6 +112,15 @@ static inline void dev_pm_disarm_wake_irq(struct wake_irq *wirq) { } +static inline void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status) +{ +} + +static inline void dev_pm_disable_wake_irq_check(struct device *dev) +{ +} + #endif #ifdef CONFIG_PM_SLEEP diff --git a/drivers/base/power/qos.c b/drivers/base/power/qos.c index 7f3646e459cb..58fcc758334e 100644 --- a/drivers/base/power/qos.c +++ b/drivers/base/power/qos.c @@ -856,7 +856,10 @@ int dev_pm_qos_update_user_latency_tolerance(struct device *dev, s32 val) struct dev_pm_qos_request *req; if (val < 0) { - ret = -EINVAL; + if (val == PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT) + ret = 0; + else + ret = -EINVAL; goto out; } req = kzalloc(sizeof(*req), GFP_KERNEL); @@ -883,6 +886,7 @@ int dev_pm_qos_update_user_latency_tolerance(struct device *dev, s32 val) mutex_unlock(&dev_pm_qos_mtx); return ret; } +EXPORT_SYMBOL_GPL(dev_pm_qos_update_user_latency_tolerance); /** * dev_pm_qos_expose_latency_tolerance - Expose latency tolerance to userspace diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 82a081ea4317..26856d050037 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -241,7 +241,8 @@ static int rpm_check_suspend_allowed(struct device *dev) retval = -EACCES; else if (atomic_read(&dev->power.usage_count) > 0) retval = -EAGAIN; - else if (!pm_children_suspended(dev)) + else if (!dev->power.ignore_children && + atomic_read(&dev->power.child_count)) retval = -EBUSY; /* Pending resume requests take precedence over suspends. */ @@ -515,7 +516,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) callback = RPM_GET_CALLBACK(dev, runtime_suspend); - dev_pm_enable_wake_irq(dev); + dev_pm_enable_wake_irq_check(dev, true); retval = rpm_callback(callback, dev); if (retval) goto fail; @@ -554,7 +555,7 @@ static int rpm_suspend(struct device *dev, int rpmflags) return retval; fail: - dev_pm_disable_wake_irq(dev); + dev_pm_disable_wake_irq_check(dev); __update_runtime_status(dev, RPM_ACTIVE); dev->power.deferred_resume = false; wake_up_all(&dev->power.wait_queue); @@ -712,8 +713,8 @@ static int rpm_resume(struct device *dev, int rpmflags) spin_lock(&parent->power.lock); /* - * We can resume if the parent's runtime PM is disabled or it - * is set to ignore children. + * Resume the parent if it has runtime PM enabled and not been + * set to ignore its children. */ if (!parent->power.disable_depth && !parent->power.ignore_children) { @@ -737,12 +738,12 @@ static int rpm_resume(struct device *dev, int rpmflags) callback = RPM_GET_CALLBACK(dev, runtime_resume); - dev_pm_disable_wake_irq(dev); + dev_pm_disable_wake_irq_check(dev); retval = rpm_callback(callback, dev); if (retval) { __update_runtime_status(dev, RPM_SUSPENDED); pm_runtime_cancel_pending(dev); - dev_pm_enable_wake_irq(dev); + dev_pm_enable_wake_irq_check(dev, false); } else { no_callback: __update_runtime_status(dev, RPM_ACTIVE); @@ -1027,7 +1028,17 @@ int __pm_runtime_set_status(struct device *dev, unsigned int status) goto out_set; if (status == RPM_SUSPENDED) { - /* It always is possible to set the status to 'suspended'. */ + /* + * It is invalid to suspend a device with an active child, + * unless it has been set to ignore its children. + */ + if (!dev->power.ignore_children && + atomic_read(&dev->power.child_count)) { + dev_err(dev, "runtime PM trying to suspend device but active child\n"); + error = -EBUSY; + goto out; + } + if (parent) { atomic_add_unless(&parent->power.child_count, -1, 0); notify_parent = !parent->power.ignore_children; @@ -1478,6 +1489,16 @@ int pm_runtime_force_suspend(struct device *dev) if (ret) goto err; + /* + * Increase the runtime PM usage count for the device's parent, in case + * when we find the device being used when system suspend was invoked. + * This informs pm_runtime_force_resume() to resume the parent + * immediately, which is needed to be able to resume its children, + * when not deferring the resume to be managed via runtime PM. + */ + if (dev->parent && atomic_read(&dev->power.usage_count) > 1) + pm_runtime_get_noresume(dev->parent); + pm_runtime_set_suspended(dev); return 0; err: @@ -1487,16 +1508,20 @@ err: EXPORT_SYMBOL_GPL(pm_runtime_force_suspend); /** - * pm_runtime_force_resume - Force a device into resume state. + * pm_runtime_force_resume - Force a device into resume state if needed. * @dev: Device to resume. * * Prior invoking this function we expect the user to have brought the device * into low power state by a call to pm_runtime_force_suspend(). Here we reverse - * those actions and brings the device into full power. We update the runtime PM - * status and re-enables runtime PM. + * those actions and brings the device into full power, if it is expected to be + * used on system resume. To distinguish that, we check whether the runtime PM + * usage count is greater than 1 (the PM core increases the usage count in the + * system PM prepare phase), as that indicates a real user (such as a subsystem, + * driver, userspace, etc.) is using it. If that is the case, the device is + * expected to be used on system resume as well, so then we resume it. In the + * other case, we defer the resume to be managed via runtime PM. * - * Typically this function may be invoked from a system resume callback to make - * sure the device is put into full power state. + * Typically this function may be invoked from a system resume callback. */ int pm_runtime_force_resume(struct device *dev) { @@ -1513,6 +1538,17 @@ int pm_runtime_force_resume(struct device *dev) if (!pm_runtime_status_suspended(dev)) goto out; + /* + * Decrease the parent's runtime PM usage count, if we increased it + * during system suspend in pm_runtime_force_suspend(). + */ + if (atomic_read(&dev->power.usage_count) > 1) { + if (dev->parent) + pm_runtime_put_noidle(dev->parent); + } else { + goto out; + } + ret = pm_runtime_set_active(dev); if (ret) goto out; diff --git a/drivers/base/power/sysfs.c b/drivers/base/power/sysfs.c index a7b46798c81d..33b4b902741a 100644 --- a/drivers/base/power/sysfs.c +++ b/drivers/base/power/sysfs.c @@ -263,7 +263,11 @@ static ssize_t pm_qos_latency_tolerance_store(struct device *dev, s32 value; int ret; - if (kstrtos32(buf, 0, &value)) { + if (kstrtos32(buf, 0, &value) == 0) { + /* Users can't write negative values directly */ + if (value < 0) + return -EINVAL; + } else { if (!strcmp(buf, "auto") || !strcmp(buf, "auto\n")) value = PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT; else if (!strcmp(buf, "any") || !strcmp(buf, "any\n")) diff --git a/drivers/base/power/wakeirq.c b/drivers/base/power/wakeirq.c index 0d77cd6fd8d1..404d94c6c8bc 100644 --- a/drivers/base/power/wakeirq.c +++ b/drivers/base/power/wakeirq.c @@ -110,8 +110,10 @@ void dev_pm_clear_wake_irq(struct device *dev) dev->power.wakeirq = NULL; spin_unlock_irqrestore(&dev->power.lock, flags); - if (wirq->dedicated_irq) + if (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED) { free_irq(wirq->irq, wirq); + wirq->status &= ~WAKE_IRQ_DEDICATED_MASK; + } kfree(wirq); } EXPORT_SYMBOL_GPL(dev_pm_clear_wake_irq); @@ -179,7 +181,6 @@ int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq) wirq->dev = dev; wirq->irq = irq; - wirq->dedicated_irq = true; irq_set_status_flags(irq, IRQ_NOAUTOEN); /* @@ -195,6 +196,8 @@ int dev_pm_set_dedicated_wake_irq(struct device *dev, int irq) if (err) goto err_free_irq; + wirq->status = WAKE_IRQ_DEDICATED_ALLOCATED; + return err; err_free_irq: @@ -210,9 +213,9 @@ EXPORT_SYMBOL_GPL(dev_pm_set_dedicated_wake_irq); * dev_pm_enable_wake_irq - Enable device wake-up interrupt * @dev: Device * - * Called from the bus code or the device driver for - * runtime_suspend() to enable the wake-up interrupt while - * the device is running. + * Optionally called from the bus code or the device driver for + * runtime_resume() to override the PM runtime core managed wake-up + * interrupt handling to enable the wake-up interrupt. * * Note that for runtime_suspend()) the wake-up interrupts * should be unconditionally enabled unlike for suspend() @@ -222,7 +225,7 @@ void dev_pm_enable_wake_irq(struct device *dev) { struct wake_irq *wirq = dev->power.wakeirq; - if (wirq && wirq->dedicated_irq) + if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)) enable_irq(wirq->irq); } EXPORT_SYMBOL_GPL(dev_pm_enable_wake_irq); @@ -231,20 +234,73 @@ EXPORT_SYMBOL_GPL(dev_pm_enable_wake_irq); * dev_pm_disable_wake_irq - Disable device wake-up interrupt * @dev: Device * - * Called from the bus code or the device driver for - * runtime_resume() to disable the wake-up interrupt while - * the device is running. + * Optionally called from the bus code or the device driver for + * runtime_suspend() to override the PM runtime core managed wake-up + * interrupt handling to disable the wake-up interrupt. */ void dev_pm_disable_wake_irq(struct device *dev) { struct wake_irq *wirq = dev->power.wakeirq; - if (wirq && wirq->dedicated_irq) + if (wirq && (wirq->status & WAKE_IRQ_DEDICATED_ALLOCATED)) disable_irq_nosync(wirq->irq); } EXPORT_SYMBOL_GPL(dev_pm_disable_wake_irq); /** + * dev_pm_enable_wake_irq_check - Checks and enables wake-up interrupt + * @dev: Device + * @can_change_status: Can change wake-up interrupt status + * + * Enables wakeirq conditionally. We need to enable wake-up interrupt + * lazily on the first rpm_suspend(). This is needed as the consumer device + * starts in RPM_SUSPENDED state, and the the first pm_runtime_get() would + * otherwise try to disable already disabled wakeirq. The wake-up interrupt + * starts disabled with IRQ_NOAUTOEN set. + * + * Should be only called from rpm_suspend() and rpm_resume() path. + * Caller must hold &dev->power.lock to change wirq->status + */ +void dev_pm_enable_wake_irq_check(struct device *dev, + bool can_change_status) +{ + struct wake_irq *wirq = dev->power.wakeirq; + + if (!wirq || !((wirq->status & WAKE_IRQ_DEDICATED_MASK))) + return; + + if (likely(wirq->status & WAKE_IRQ_DEDICATED_MANAGED)) { + goto enable; + } else if (can_change_status) { + wirq->status |= WAKE_IRQ_DEDICATED_MANAGED; + goto enable; + } + + return; + +enable: + enable_irq(wirq->irq); +} + +/** + * dev_pm_disable_wake_irq_check - Checks and disables wake-up interrupt + * @dev: Device + * + * Disables wake-up interrupt conditionally based on status. + * Should be only called from rpm_suspend() and rpm_resume() path. + */ +void dev_pm_disable_wake_irq_check(struct device *dev) +{ + struct wake_irq *wirq = dev->power.wakeirq; + + if (!wirq || !((wirq->status & WAKE_IRQ_DEDICATED_MASK))) + return; + + if (wirq->status & WAKE_IRQ_DEDICATED_MANAGED) + disable_irq_nosync(wirq->irq); +} + +/** * dev_pm_arm_wake_irq - Arm device wake-up * @wirq: Device wake-up interrupt * diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index 62e4de2aa8d1..bf9ba26981a5 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -811,7 +811,7 @@ void pm_print_active_wakeup_sources(void) rcu_read_lock(); list_for_each_entry_rcu(ws, &wakeup_sources, entry) { if (ws->active) { - pr_info("active wakeup source: %s\n", ws->name); + pr_debug("active wakeup source: %s\n", ws->name); active = 1; } else if (!active && (!last_activity_ws || @@ -822,7 +822,7 @@ void pm_print_active_wakeup_sources(void) } if (!active && last_activity_ws) - pr_info("last active wakeup source: %s\n", + pr_debug("last active wakeup source: %s\n", last_activity_ws->name); rcu_read_unlock(); } @@ -905,7 +905,7 @@ bool pm_get_wakeup_count(unsigned int *count, bool block) split_counters(&cnt, &inpr); if (inpr == 0 || signal_pending(current)) break; - + pm_print_active_wakeup_sources(); schedule(); } finish_wait(&wakeup_count_wait_queue, &wait); diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm index d89b8afe23b6..920c469f3953 100644 --- a/drivers/cpufreq/Kconfig.arm +++ b/drivers/cpufreq/Kconfig.arm @@ -12,6 +12,27 @@ config ARM_BIG_LITTLE_CPUFREQ help This enables the Generic CPUfreq driver for ARM big.LITTLE platforms. +config ARM_BRCMSTB_AVS_CPUFREQ + tristate "Broadcom STB AVS CPUfreq driver" + depends on ARCH_BRCMSTB || COMPILE_TEST + default y + help + Some Broadcom STB SoCs use a co-processor running proprietary firmware + ("AVS") to handle voltage and frequency scaling. This driver provides + a standard CPUfreq interface to to the firmware. + + Say Y, if you have a Broadcom SoC with AVS support for DFS or DVFS. + +config ARM_BRCMSTB_AVS_CPUFREQ_DEBUG + bool "Broadcom STB AVS CPUfreq driver sysfs debug capability" + depends on ARM_BRCMSTB_AVS_CPUFREQ + help + Enabling this option turns on debug support via sysfs under + /sys/kernel/debug/brcmstb-avs-cpufreq. It is possible to read all and + write some AVS mailbox registers through sysfs entries. + + If in doubt, say N. + config ARM_DT_BL_CPUFREQ tristate "Generic probing via DT for ARM big LITTLE CPUfreq driver" depends on ARM_BIG_LITTLE_CPUFREQ && OF @@ -60,14 +81,6 @@ config ARM_IMX6Q_CPUFREQ If in doubt, say N. -config ARM_INTEGRATOR - tristate "CPUfreq driver for ARM Integrator CPUs" - depends on ARCH_INTEGRATOR - default y - help - This enables the CPUfreq driver for ARM Integrator CPUs. - If in doubt, say Y. - config ARM_KIRKWOOD_CPUFREQ def_bool MACH_KIRKWOOD help diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile index 0a9b6a093646..1e46c3918e7a 100644 --- a/drivers/cpufreq/Makefile +++ b/drivers/cpufreq/Makefile @@ -51,12 +51,12 @@ obj-$(CONFIG_ARM_BIG_LITTLE_CPUFREQ) += arm_big_little.o # LITTLE drivers, so that it is probed last. obj-$(CONFIG_ARM_DT_BL_CPUFREQ) += arm_big_little_dt.o +obj-$(CONFIG_ARM_BRCMSTB_AVS_CPUFREQ) += brcmstb-avs-cpufreq.o obj-$(CONFIG_ARCH_DAVINCI) += davinci-cpufreq.o obj-$(CONFIG_UX500_SOC_DB8500) += dbx500-cpufreq.o obj-$(CONFIG_ARM_EXYNOS5440_CPUFREQ) += exynos5440-cpufreq.o obj-$(CONFIG_ARM_HIGHBANK_CPUFREQ) += highbank-cpufreq.o obj-$(CONFIG_ARM_IMX6Q_CPUFREQ) += imx6q-cpufreq.o -obj-$(CONFIG_ARM_INTEGRATOR) += integrator-cpufreq.o obj-$(CONFIG_ARM_KIRKWOOD_CPUFREQ) += kirkwood-cpufreq.o obj-$(CONFIG_ARM_MT8173_CPUFREQ) += mt8173-cpufreq.o obj-$(CONFIG_ARM_OMAP2PLUS_CPUFREQ) += omap-cpufreq.o diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 297e9128fe9f..3a98702b7445 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -84,7 +84,6 @@ static inline struct acpi_processor_performance *to_perf_data(struct acpi_cpufre static struct cpufreq_driver acpi_cpufreq_driver; static unsigned int acpi_pstate_strict; -static struct msr __percpu *msrs; static bool boost_state(unsigned int cpu) { @@ -104,11 +103,10 @@ static bool boost_state(unsigned int cpu) return false; } -static void boost_set_msrs(bool enable, const struct cpumask *cpumask) +static int boost_set_msr(bool enable) { - u32 cpu; u32 msr_addr; - u64 msr_mask; + u64 msr_mask, val; switch (boot_cpu_data.x86_vendor) { case X86_VENDOR_INTEL: @@ -120,26 +118,31 @@ static void boost_set_msrs(bool enable, const struct cpumask *cpumask) msr_mask = MSR_K7_HWCR_CPB_DIS; break; default: - return; + return -EINVAL; } - rdmsr_on_cpus(cpumask, msr_addr, msrs); + rdmsrl(msr_addr, val); - for_each_cpu(cpu, cpumask) { - struct msr *reg = per_cpu_ptr(msrs, cpu); - if (enable) - reg->q &= ~msr_mask; - else - reg->q |= msr_mask; - } + if (enable) + val &= ~msr_mask; + else + val |= msr_mask; + + wrmsrl(msr_addr, val); + return 0; +} + +static void boost_set_msr_each(void *p_en) +{ + bool enable = (bool) p_en; - wrmsr_on_cpus(cpumask, msr_addr, msrs); + boost_set_msr(enable); } static int set_boost(int val) { get_online_cpus(); - boost_set_msrs(val, cpu_online_mask); + on_each_cpu(boost_set_msr_each, (void *)(long)val, 1); put_online_cpus(); pr_debug("Core Boosting %sabled.\n", val ? "en" : "dis"); @@ -536,46 +539,24 @@ static void free_acpi_perf_data(void) free_percpu(acpi_perf_data); } -static int boost_notify(struct notifier_block *nb, unsigned long action, - void *hcpu) +static int cpufreq_boost_online(unsigned int cpu) { - unsigned cpu = (long)hcpu; - const struct cpumask *cpumask; - - cpumask = get_cpu_mask(cpu); + /* + * On the CPU_UP path we simply keep the boost-disable flag + * in sync with the current global state. + */ + return boost_set_msr(acpi_cpufreq_driver.boost_enabled); +} +static int cpufreq_boost_down_prep(unsigned int cpu) +{ /* * Clear the boost-disable bit on the CPU_DOWN path so that - * this cpu cannot block the remaining ones from boosting. On - * the CPU_UP path we simply keep the boost-disable flag in - * sync with the current global state. + * this cpu cannot block the remaining ones from boosting. */ - - switch (action) { - case CPU_DOWN_FAILED: - case CPU_DOWN_FAILED_FROZEN: - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - boost_set_msrs(acpi_cpufreq_driver.boost_enabled, cpumask); - break; - - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - boost_set_msrs(1, cpumask); - break; - - default: - break; - } - - return NOTIFY_OK; + return boost_set_msr(1); } - -static struct notifier_block boost_nb = { - .notifier_call = boost_notify, -}; - /* * acpi_cpufreq_early_init - initialize ACPI P-States library * @@ -922,37 +903,35 @@ static struct cpufreq_driver acpi_cpufreq_driver = { .attr = acpi_cpufreq_attr, }; +static enum cpuhp_state acpi_cpufreq_online; + static void __init acpi_cpufreq_boost_init(void) { - if (boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA)) { - msrs = msrs_alloc(); - - if (!msrs) - return; - - acpi_cpufreq_driver.set_boost = set_boost; - acpi_cpufreq_driver.boost_enabled = boost_state(0); - - cpu_notifier_register_begin(); + int ret; - /* Force all MSRs to the same value */ - boost_set_msrs(acpi_cpufreq_driver.boost_enabled, - cpu_online_mask); + if (!(boot_cpu_has(X86_FEATURE_CPB) || boot_cpu_has(X86_FEATURE_IDA))) + return; - __register_cpu_notifier(&boost_nb); + acpi_cpufreq_driver.set_boost = set_boost; + acpi_cpufreq_driver.boost_enabled = boost_state(0); - cpu_notifier_register_done(); + /* + * This calls the online callback on all online cpu and forces all + * MSRs to the same value. + */ + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "cpufreq/acpi:online", + cpufreq_boost_online, cpufreq_boost_down_prep); + if (ret < 0) { + pr_err("acpi_cpufreq: failed to register hotplug callbacks\n"); + return; } + acpi_cpufreq_online = ret; } static void acpi_cpufreq_boost_exit(void) { - if (msrs) { - unregister_cpu_notifier(&boost_nb); - - msrs_free(msrs); - msrs = NULL; - } + if (acpi_cpufreq_online >= 0) + cpuhp_remove_state_nocalls(acpi_cpufreq_online); } static int __init acpi_cpufreq_init(void) diff --git a/drivers/cpufreq/brcmstb-avs-cpufreq.c b/drivers/cpufreq/brcmstb-avs-cpufreq.c new file mode 100644 index 000000000000..4fda623e55bb --- /dev/null +++ b/drivers/cpufreq/brcmstb-avs-cpufreq.c @@ -0,0 +1,1057 @@ +/* + * CPU frequency scaling for Broadcom SoCs with AVS firmware that + * supports DVS or DVFS + * + * Copyright (c) 2016 Broadcom + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation version 2. + * + * This program is distributed "as is" WITHOUT ANY WARRANTY of any + * kind, whether express or implied; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +/* + * "AVS" is the name of a firmware developed at Broadcom. It derives + * its name from the technique called "Adaptive Voltage Scaling". + * Adaptive voltage scaling was the original purpose of this firmware. + * The AVS firmware still supports "AVS mode", where all it does is + * adaptive voltage scaling. However, on some newer Broadcom SoCs, the + * AVS Firmware, despite its unchanged name, also supports DFS mode and + * DVFS mode. + * + * In the context of this document and the related driver, "AVS" by + * itself always means the Broadcom firmware and never refers to the + * technique called "Adaptive Voltage Scaling". + * + * The Broadcom STB AVS CPUfreq driver provides voltage and frequency + * scaling on Broadcom SoCs using AVS firmware with support for DFS and + * DVFS. The AVS firmware is running on its own co-processor. The + * driver supports both uniprocessor (UP) and symmetric multiprocessor + * (SMP) systems which share clock and voltage across all CPUs. + * + * Actual voltage and frequency scaling is done solely by the AVS + * firmware. This driver does not change frequency or voltage itself. + * It provides a standard CPUfreq interface to the rest of the kernel + * and to userland. It interfaces with the AVS firmware to effect the + * requested changes and to report back the current system status in a + * way that is expected by existing tools. + */ + +#include <linux/cpufreq.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/platform_device.h> +#include <linux/semaphore.h> + +#ifdef CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG +#include <linux/ctype.h> +#include <linux/debugfs.h> +#include <linux/slab.h> +#include <linux/uaccess.h> +#endif + +/* Max number of arguments AVS calls take */ +#define AVS_MAX_CMD_ARGS 4 +/* + * This macro is used to generate AVS parameter register offsets. For + * x >= AVS_MAX_CMD_ARGS, it returns 0 to protect against accidental memory + * access outside of the parameter range. (Offset 0 is the first parameter.) + */ +#define AVS_PARAM_MULT(x) ((x) < AVS_MAX_CMD_ARGS ? (x) : 0) + +/* AVS Mailbox Register offsets */ +#define AVS_MBOX_COMMAND 0x00 +#define AVS_MBOX_STATUS 0x04 +#define AVS_MBOX_VOLTAGE0 0x08 +#define AVS_MBOX_TEMP0 0x0c +#define AVS_MBOX_PV0 0x10 +#define AVS_MBOX_MV0 0x14 +#define AVS_MBOX_PARAM(x) (0x18 + AVS_PARAM_MULT(x) * sizeof(u32)) +#define AVS_MBOX_REVISION 0x28 +#define AVS_MBOX_PSTATE 0x2c +#define AVS_MBOX_HEARTBEAT 0x30 +#define AVS_MBOX_MAGIC 0x34 +#define AVS_MBOX_SIGMA_HVT 0x38 +#define AVS_MBOX_SIGMA_SVT 0x3c +#define AVS_MBOX_VOLTAGE1 0x40 +#define AVS_MBOX_TEMP1 0x44 +#define AVS_MBOX_PV1 0x48 +#define AVS_MBOX_MV1 0x4c +#define AVS_MBOX_FREQUENCY 0x50 + +/* AVS Commands */ +#define AVS_CMD_AVAILABLE 0x00 +#define AVS_CMD_DISABLE 0x10 +#define AVS_CMD_ENABLE 0x11 +#define AVS_CMD_S2_ENTER 0x12 +#define AVS_CMD_S2_EXIT 0x13 +#define AVS_CMD_BBM_ENTER 0x14 +#define AVS_CMD_BBM_EXIT 0x15 +#define AVS_CMD_S3_ENTER 0x16 +#define AVS_CMD_S3_EXIT 0x17 +#define AVS_CMD_BALANCE 0x18 +/* PMAP and P-STATE commands */ +#define AVS_CMD_GET_PMAP 0x30 +#define AVS_CMD_SET_PMAP 0x31 +#define AVS_CMD_GET_PSTATE 0x40 +#define AVS_CMD_SET_PSTATE 0x41 + +/* Different modes AVS supports (for GET_PMAP/SET_PMAP) */ +#define AVS_MODE_AVS 0x0 +#define AVS_MODE_DFS 0x1 +#define AVS_MODE_DVS 0x2 +#define AVS_MODE_DVFS 0x3 + +/* + * PMAP parameter p1 + * unused:31-24, mdiv_p0:23-16, unused:15-14, pdiv:13-10 , ndiv_int:9-0 + */ +#define NDIV_INT_SHIFT 0 +#define NDIV_INT_MASK 0x3ff +#define PDIV_SHIFT 10 +#define PDIV_MASK 0xf +#define MDIV_P0_SHIFT 16 +#define MDIV_P0_MASK 0xff +/* + * PMAP parameter p2 + * mdiv_p4:31-24, mdiv_p3:23-16, mdiv_p2:15:8, mdiv_p1:7:0 + */ +#define MDIV_P1_SHIFT 0 +#define MDIV_P1_MASK 0xff +#define MDIV_P2_SHIFT 8 +#define MDIV_P2_MASK 0xff +#define MDIV_P3_SHIFT 16 +#define MDIV_P3_MASK 0xff +#define MDIV_P4_SHIFT 24 +#define MDIV_P4_MASK 0xff + +/* Different P-STATES AVS supports (for GET_PSTATE/SET_PSTATE) */ +#define AVS_PSTATE_P0 0x0 +#define AVS_PSTATE_P1 0x1 +#define AVS_PSTATE_P2 0x2 +#define AVS_PSTATE_P3 0x3 +#define AVS_PSTATE_P4 0x4 +#define AVS_PSTATE_MAX AVS_PSTATE_P4 + +/* CPU L2 Interrupt Controller Registers */ +#define AVS_CPU_L2_SET0 0x04 +#define AVS_CPU_L2_INT_MASK BIT(31) + +/* AVS Command Status Values */ +#define AVS_STATUS_CLEAR 0x00 +/* Command/notification accepted */ +#define AVS_STATUS_SUCCESS 0xf0 +/* Command/notification rejected */ +#define AVS_STATUS_FAILURE 0xff +/* Invalid command/notification (unknown) */ +#define AVS_STATUS_INVALID 0xf1 +/* Non-AVS modes are not supported */ +#define AVS_STATUS_NO_SUPP 0xf2 +/* Cannot set P-State until P-Map supplied */ +#define AVS_STATUS_NO_MAP 0xf3 +/* Cannot change P-Map after initial P-Map set */ +#define AVS_STATUS_MAP_SET 0xf4 +/* Max AVS status; higher numbers are used for debugging */ +#define AVS_STATUS_MAX 0xff + +/* Other AVS related constants */ +#define AVS_LOOP_LIMIT 10000 +#define AVS_TIMEOUT 300 /* in ms; expected completion is < 10ms */ +#define AVS_FIRMWARE_MAGIC 0xa11600d1 + +#define BRCM_AVS_CPUFREQ_PREFIX "brcmstb-avs" +#define BRCM_AVS_CPUFREQ_NAME BRCM_AVS_CPUFREQ_PREFIX "-cpufreq" +#define BRCM_AVS_CPU_DATA "brcm,avs-cpu-data-mem" +#define BRCM_AVS_CPU_INTR "brcm,avs-cpu-l2-intr" +#define BRCM_AVS_HOST_INTR "sw_intr" + +struct pmap { + unsigned int mode; + unsigned int p1; + unsigned int p2; + unsigned int state; +}; + +struct private_data { + void __iomem *base; + void __iomem *avs_intr_base; + struct device *dev; +#ifdef CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG + struct dentry *debugfs; +#endif + struct completion done; + struct semaphore sem; + struct pmap pmap; +}; + +#ifdef CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG + +enum debugfs_format { + DEBUGFS_NORMAL, + DEBUGFS_FLOAT, + DEBUGFS_REV, +}; + +struct debugfs_data { + struct debugfs_entry *entry; + struct private_data *priv; +}; + +struct debugfs_entry { + char *name; + u32 offset; + fmode_t mode; + enum debugfs_format format; +}; + +#define DEBUGFS_ENTRY(name, mode, format) { \ + #name, AVS_MBOX_##name, mode, format \ +} + +/* + * These are used for debugfs only. Otherwise we use AVS_MBOX_PARAM() directly. + */ +#define AVS_MBOX_PARAM1 AVS_MBOX_PARAM(0) +#define AVS_MBOX_PARAM2 AVS_MBOX_PARAM(1) +#define AVS_MBOX_PARAM3 AVS_MBOX_PARAM(2) +#define AVS_MBOX_PARAM4 AVS_MBOX_PARAM(3) + +/* + * This table stores the name, access permissions and offset for each hardware + * register and is used to generate debugfs entries. + */ +static struct debugfs_entry debugfs_entries[] = { + DEBUGFS_ENTRY(COMMAND, S_IWUSR, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(STATUS, S_IWUSR, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(VOLTAGE0, 0, DEBUGFS_FLOAT), + DEBUGFS_ENTRY(TEMP0, 0, DEBUGFS_FLOAT), + DEBUGFS_ENTRY(PV0, 0, DEBUGFS_FLOAT), + DEBUGFS_ENTRY(MV0, 0, DEBUGFS_FLOAT), + DEBUGFS_ENTRY(PARAM1, S_IWUSR, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(PARAM2, S_IWUSR, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(PARAM3, S_IWUSR, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(PARAM4, S_IWUSR, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(REVISION, 0, DEBUGFS_REV), + DEBUGFS_ENTRY(PSTATE, 0, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(HEARTBEAT, 0, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(MAGIC, S_IWUSR, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(SIGMA_HVT, 0, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(SIGMA_SVT, 0, DEBUGFS_NORMAL), + DEBUGFS_ENTRY(VOLTAGE1, 0, DEBUGFS_FLOAT), + DEBUGFS_ENTRY(TEMP1, 0, DEBUGFS_FLOAT), + DEBUGFS_ENTRY(PV1, 0, DEBUGFS_FLOAT), + DEBUGFS_ENTRY(MV1, 0, DEBUGFS_FLOAT), + DEBUGFS_ENTRY(FREQUENCY, 0, DEBUGFS_NORMAL), +}; + +static int brcm_avs_target_index(struct cpufreq_policy *, unsigned int); + +static char *__strtolower(char *s) +{ + char *p; + + for (p = s; *p; p++) + *p = tolower(*p); + + return s; +} + +#endif /* CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG */ + +static void __iomem *__map_region(const char *name) +{ + struct device_node *np; + void __iomem *ptr; + + np = of_find_compatible_node(NULL, NULL, name); + if (!np) + return NULL; + + ptr = of_iomap(np, 0); + of_node_put(np); + + return ptr; +} + +static int __issue_avs_command(struct private_data *priv, int cmd, bool is_send, + u32 args[]) +{ + unsigned long time_left = msecs_to_jiffies(AVS_TIMEOUT); + void __iomem *base = priv->base; + unsigned int i; + int ret; + u32 val; + + ret = down_interruptible(&priv->sem); + if (ret) + return ret; + + /* + * Make sure no other command is currently running: cmd is 0 if AVS + * co-processor is idle. Due to the guard above, we should almost never + * have to wait here. + */ + for (i = 0, val = 1; val != 0 && i < AVS_LOOP_LIMIT; i++) + val = readl(base + AVS_MBOX_COMMAND); + + /* Give the caller a chance to retry if AVS is busy. */ + if (i == AVS_LOOP_LIMIT) { + ret = -EAGAIN; + goto out; + } + + /* Clear status before we begin. */ + writel(AVS_STATUS_CLEAR, base + AVS_MBOX_STATUS); + + /* We need to send arguments for this command. */ + if (args && is_send) { + for (i = 0; i < AVS_MAX_CMD_ARGS; i++) + writel(args[i], base + AVS_MBOX_PARAM(i)); + } + + /* Protect from spurious interrupts. */ + reinit_completion(&priv->done); + + /* Now issue the command & tell firmware to wake up to process it. */ + writel(cmd, base + AVS_MBOX_COMMAND); + writel(AVS_CPU_L2_INT_MASK, priv->avs_intr_base + AVS_CPU_L2_SET0); + + /* Wait for AVS co-processor to finish processing the command. */ + time_left = wait_for_completion_timeout(&priv->done, time_left); + + /* + * If the AVS status is not in the expected range, it means AVS didn't + * complete our command in time, and we return an error. Also, if there + * is no "time left", we timed out waiting for the interrupt. + */ + val = readl(base + AVS_MBOX_STATUS); + if (time_left == 0 || val == 0 || val > AVS_STATUS_MAX) { + dev_err(priv->dev, "AVS command %#x didn't complete in time\n", + cmd); + dev_err(priv->dev, " Time left: %u ms, AVS status: %#x\n", + jiffies_to_msecs(time_left), val); + ret = -ETIMEDOUT; + goto out; + } + + /* This command returned arguments, so we read them back. */ + if (args && !is_send) { + for (i = 0; i < AVS_MAX_CMD_ARGS; i++) + args[i] = readl(base + AVS_MBOX_PARAM(i)); + } + + /* Clear status to tell AVS co-processor we are done. */ + writel(AVS_STATUS_CLEAR, base + AVS_MBOX_STATUS); + + /* Convert firmware errors to errno's as much as possible. */ + switch (val) { + case AVS_STATUS_INVALID: + ret = -EINVAL; + break; + case AVS_STATUS_NO_SUPP: + ret = -ENOTSUPP; + break; + case AVS_STATUS_NO_MAP: + ret = -ENOENT; + break; + case AVS_STATUS_MAP_SET: + ret = -EEXIST; + break; + case AVS_STATUS_FAILURE: + ret = -EIO; + break; + } + +out: + up(&priv->sem); + + return ret; +} + +static irqreturn_t irq_handler(int irq, void *data) +{ + struct private_data *priv = data; + + /* AVS command completed execution. Wake up __issue_avs_command(). */ + complete(&priv->done); + + return IRQ_HANDLED; +} + +static char *brcm_avs_mode_to_string(unsigned int mode) +{ + switch (mode) { + case AVS_MODE_AVS: + return "AVS"; + case AVS_MODE_DFS: + return "DFS"; + case AVS_MODE_DVS: + return "DVS"; + case AVS_MODE_DVFS: + return "DVFS"; + } + return NULL; +} + +static void brcm_avs_parse_p1(u32 p1, unsigned int *mdiv_p0, unsigned int *pdiv, + unsigned int *ndiv) +{ + *mdiv_p0 = (p1 >> MDIV_P0_SHIFT) & MDIV_P0_MASK; + *pdiv = (p1 >> PDIV_SHIFT) & PDIV_MASK; + *ndiv = (p1 >> NDIV_INT_SHIFT) & NDIV_INT_MASK; +} + +static void brcm_avs_parse_p2(u32 p2, unsigned int *mdiv_p1, + unsigned int *mdiv_p2, unsigned int *mdiv_p3, + unsigned int *mdiv_p4) +{ + *mdiv_p4 = (p2 >> MDIV_P4_SHIFT) & MDIV_P4_MASK; + *mdiv_p3 = (p2 >> MDIV_P3_SHIFT) & MDIV_P3_MASK; + *mdiv_p2 = (p2 >> MDIV_P2_SHIFT) & MDIV_P2_MASK; + *mdiv_p1 = (p2 >> MDIV_P1_SHIFT) & MDIV_P1_MASK; +} + +static int brcm_avs_get_pmap(struct private_data *priv, struct pmap *pmap) +{ + u32 args[AVS_MAX_CMD_ARGS]; + int ret; + + ret = __issue_avs_command(priv, AVS_CMD_GET_PMAP, false, args); + if (ret || !pmap) + return ret; + + pmap->mode = args[0]; + pmap->p1 = args[1]; + pmap->p2 = args[2]; + pmap->state = args[3]; + + return 0; +} + +static int brcm_avs_set_pmap(struct private_data *priv, struct pmap *pmap) +{ + u32 args[AVS_MAX_CMD_ARGS]; + + args[0] = pmap->mode; + args[1] = pmap->p1; + args[2] = pmap->p2; + args[3] = pmap->state; + + return __issue_avs_command(priv, AVS_CMD_SET_PMAP, true, args); +} + +static int brcm_avs_get_pstate(struct private_data *priv, unsigned int *pstate) +{ + u32 args[AVS_MAX_CMD_ARGS]; + int ret; + + ret = __issue_avs_command(priv, AVS_CMD_GET_PSTATE, false, args); + if (ret) + return ret; + *pstate = args[0]; + + return 0; +} + +static int brcm_avs_set_pstate(struct private_data *priv, unsigned int pstate) +{ + u32 args[AVS_MAX_CMD_ARGS]; + + args[0] = pstate; + + return __issue_avs_command(priv, AVS_CMD_SET_PSTATE, true, args); +} + +static unsigned long brcm_avs_get_voltage(void __iomem *base) +{ + return readl(base + AVS_MBOX_VOLTAGE1); +} + +static unsigned long brcm_avs_get_frequency(void __iomem *base) +{ + return readl(base + AVS_MBOX_FREQUENCY) * 1000; /* in kHz */ +} + +/* + * We determine which frequencies are supported by cycling through all P-states + * and reading back what frequency we are running at for each P-state. + */ +static struct cpufreq_frequency_table * +brcm_avs_get_freq_table(struct device *dev, struct private_data *priv) +{ + struct cpufreq_frequency_table *table; + unsigned int pstate; + int i, ret; + + /* Remember P-state for later */ + ret = brcm_avs_get_pstate(priv, &pstate); + if (ret) + return ERR_PTR(ret); + + table = devm_kzalloc(dev, (AVS_PSTATE_MAX + 1) * sizeof(*table), + GFP_KERNEL); + if (!table) + return ERR_PTR(-ENOMEM); + + for (i = AVS_PSTATE_P0; i <= AVS_PSTATE_MAX; i++) { + ret = brcm_avs_set_pstate(priv, i); + if (ret) + return ERR_PTR(ret); + table[i].frequency = brcm_avs_get_frequency(priv->base); + table[i].driver_data = i; + } + table[i].frequency = CPUFREQ_TABLE_END; + + /* Restore P-state */ + ret = brcm_avs_set_pstate(priv, pstate); + if (ret) + return ERR_PTR(ret); + + return table; +} + +#ifdef CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG + +#define MANT(x) (unsigned int)(abs((x)) / 1000) +#define FRAC(x) (unsigned int)(abs((x)) - abs((x)) / 1000 * 1000) + +static int brcm_avs_debug_show(struct seq_file *s, void *data) +{ + struct debugfs_data *dbgfs = s->private; + void __iomem *base; + u32 val, offset; + + if (!dbgfs) { + seq_puts(s, "No device pointer\n"); + return 0; + } + + base = dbgfs->priv->base; + offset = dbgfs->entry->offset; + val = readl(base + offset); + switch (dbgfs->entry->format) { + case DEBUGFS_NORMAL: + seq_printf(s, "%u\n", val); + break; + case DEBUGFS_FLOAT: + seq_printf(s, "%d.%03d\n", MANT(val), FRAC(val)); + break; + case DEBUGFS_REV: + seq_printf(s, "%c.%c.%c.%c\n", (val >> 24 & 0xff), + (val >> 16 & 0xff), (val >> 8 & 0xff), + val & 0xff); + break; + } + seq_printf(s, "0x%08x\n", val); + + return 0; +} + +#undef MANT +#undef FRAC + +static ssize_t brcm_avs_seq_write(struct file *file, const char __user *buf, + size_t size, loff_t *ppos) +{ + struct seq_file *s = file->private_data; + struct debugfs_data *dbgfs = s->private; + struct private_data *priv = dbgfs->priv; + void __iomem *base, *avs_intr_base; + bool use_issue_command = false; + unsigned long val, offset; + char str[128]; + int ret; + char *str_ptr = str; + + if (size >= sizeof(str)) + return -E2BIG; + + memset(str, 0, sizeof(str)); + ret = copy_from_user(str, buf, size); + if (ret) + return ret; + + base = priv->base; + avs_intr_base = priv->avs_intr_base; + offset = dbgfs->entry->offset; + /* + * Special case writing to "command" entry only: if the string starts + * with a 'c', we use the driver's __issue_avs_command() function. + * Otherwise, we perform a raw write. This should allow testing of raw + * access as well as using the higher level function. (Raw access + * doesn't clear the firmware return status after issuing the command.) + */ + if (str_ptr[0] == 'c' && offset == AVS_MBOX_COMMAND) { + use_issue_command = true; + str_ptr++; + } + if (kstrtoul(str_ptr, 0, &val) != 0) + return -EINVAL; + + /* + * Setting the P-state is a special case. We need to update the CPU + * frequency we report. + */ + if (val == AVS_CMD_SET_PSTATE) { + struct cpufreq_policy *policy; + unsigned int pstate; + + policy = cpufreq_cpu_get(smp_processor_id()); + /* Read back the P-state we are about to set */ + pstate = readl(base + AVS_MBOX_PARAM(0)); + if (use_issue_command) { + ret = brcm_avs_target_index(policy, pstate); + return ret ? ret : size; + } + policy->cur = policy->freq_table[pstate].frequency; + } + + if (use_issue_command) { + ret = __issue_avs_command(priv, val, false, NULL); + } else { + /* Locking here is not perfect, but is only for debug. */ + ret = down_interruptible(&priv->sem); + if (ret) + return ret; + + writel(val, base + offset); + /* We have to wake up the firmware to process a command. */ + if (offset == AVS_MBOX_COMMAND) + writel(AVS_CPU_L2_INT_MASK, + avs_intr_base + AVS_CPU_L2_SET0); + up(&priv->sem); + } + + return ret ? ret : size; +} + +static struct debugfs_entry *__find_debugfs_entry(const char *name) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(debugfs_entries); i++) + if (strcasecmp(debugfs_entries[i].name, name) == 0) + return &debugfs_entries[i]; + + return NULL; +} + +static int brcm_avs_debug_open(struct inode *inode, struct file *file) +{ + struct debugfs_data *data; + fmode_t fmode; + int ret; + + /* + * seq_open(), which is called by single_open(), clears "write" access. + * We need write access to some files, so we preserve our access mode + * and restore it. + */ + fmode = file->f_mode; + /* + * Check access permissions even for root. We don't want to be writing + * to read-only registers. Access for regular users has already been + * checked by the VFS layer. + */ + if ((fmode & FMODE_WRITER) && !(inode->i_mode & S_IWUSR)) + return -EACCES; + + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + /* + * We use the same file system operations for all our debug files. To + * produce specific output, we look up the file name upon opening a + * debugfs entry and map it to a memory offset. This offset is then used + * in the generic "show" function to read a specific register. + */ + data->entry = __find_debugfs_entry(file->f_path.dentry->d_iname); + data->priv = inode->i_private; + + ret = single_open(file, brcm_avs_debug_show, data); + if (ret) + kfree(data); + file->f_mode = fmode; + + return ret; +} + +static int brcm_avs_debug_release(struct inode *inode, struct file *file) +{ + struct seq_file *seq_priv = file->private_data; + struct debugfs_data *data = seq_priv->private; + + kfree(data); + return single_release(inode, file); +} + +static const struct file_operations brcm_avs_debug_ops = { + .open = brcm_avs_debug_open, + .read = seq_read, + .write = brcm_avs_seq_write, + .llseek = seq_lseek, + .release = brcm_avs_debug_release, +}; + +static void brcm_avs_cpufreq_debug_init(struct platform_device *pdev) +{ + struct private_data *priv = platform_get_drvdata(pdev); + struct dentry *dir; + int i; + + if (!priv) + return; + + dir = debugfs_create_dir(BRCM_AVS_CPUFREQ_NAME, NULL); + if (IS_ERR_OR_NULL(dir)) + return; + priv->debugfs = dir; + + for (i = 0; i < ARRAY_SIZE(debugfs_entries); i++) { + /* + * The DEBUGFS_ENTRY macro generates uppercase strings. We + * convert them to lowercase before creating the debugfs + * entries. + */ + char *entry = __strtolower(debugfs_entries[i].name); + fmode_t mode = debugfs_entries[i].mode; + + if (!debugfs_create_file(entry, S_IFREG | S_IRUGO | mode, + dir, priv, &brcm_avs_debug_ops)) { + priv->debugfs = NULL; + debugfs_remove_recursive(dir); + break; + } + } +} + +static void brcm_avs_cpufreq_debug_exit(struct platform_device *pdev) +{ + struct private_data *priv = platform_get_drvdata(pdev); + + if (priv && priv->debugfs) { + debugfs_remove_recursive(priv->debugfs); + priv->debugfs = NULL; + } +} + +#else + +static void brcm_avs_cpufreq_debug_init(struct platform_device *pdev) {} +static void brcm_avs_cpufreq_debug_exit(struct platform_device *pdev) {} + +#endif /* CONFIG_ARM_BRCMSTB_AVS_CPUFREQ_DEBUG */ + +/* + * To ensure the right firmware is running we need to + * - check the MAGIC matches what we expect + * - brcm_avs_get_pmap() doesn't return -ENOTSUPP or -EINVAL + * We need to set up our interrupt handling before calling brcm_avs_get_pmap()! + */ +static bool brcm_avs_is_firmware_loaded(struct private_data *priv) +{ + u32 magic; + int rc; + + rc = brcm_avs_get_pmap(priv, NULL); + magic = readl(priv->base + AVS_MBOX_MAGIC); + + return (magic == AVS_FIRMWARE_MAGIC) && (rc != -ENOTSUPP) && + (rc != -EINVAL); +} + +static unsigned int brcm_avs_cpufreq_get(unsigned int cpu) +{ + struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); + struct private_data *priv = policy->driver_data; + + return brcm_avs_get_frequency(priv->base); +} + +static int brcm_avs_target_index(struct cpufreq_policy *policy, + unsigned int index) +{ + return brcm_avs_set_pstate(policy->driver_data, + policy->freq_table[index].driver_data); +} + +static int brcm_avs_suspend(struct cpufreq_policy *policy) +{ + struct private_data *priv = policy->driver_data; + + return brcm_avs_get_pmap(priv, &priv->pmap); +} + +static int brcm_avs_resume(struct cpufreq_policy *policy) +{ + struct private_data *priv = policy->driver_data; + int ret; + + ret = brcm_avs_set_pmap(priv, &priv->pmap); + if (ret == -EEXIST) { + struct platform_device *pdev = cpufreq_get_driver_data(); + struct device *dev = &pdev->dev; + + dev_warn(dev, "PMAP was already set\n"); + ret = 0; + } + + return ret; +} + +/* + * All initialization code that we only want to execute once goes here. Setup + * code that can be re-tried on every core (if it failed before) can go into + * brcm_avs_cpufreq_init(). + */ +static int brcm_avs_prepare_init(struct platform_device *pdev) +{ + struct private_data *priv; + struct device *dev; + int host_irq, ret; + + dev = &pdev->dev; + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + sema_init(&priv->sem, 1); + init_completion(&priv->done); + platform_set_drvdata(pdev, priv); + + priv->base = __map_region(BRCM_AVS_CPU_DATA); + if (!priv->base) { + dev_err(dev, "Couldn't find property %s in device tree.\n", + BRCM_AVS_CPU_DATA); + return -ENOENT; + } + + priv->avs_intr_base = __map_region(BRCM_AVS_CPU_INTR); + if (!priv->avs_intr_base) { + dev_err(dev, "Couldn't find property %s in device tree.\n", + BRCM_AVS_CPU_INTR); + ret = -ENOENT; + goto unmap_base; + } + + host_irq = platform_get_irq_byname(pdev, BRCM_AVS_HOST_INTR); + if (host_irq < 0) { + dev_err(dev, "Couldn't find interrupt %s -- %d\n", + BRCM_AVS_HOST_INTR, host_irq); + ret = host_irq; + goto unmap_intr_base; + } + + ret = devm_request_irq(dev, host_irq, irq_handler, IRQF_TRIGGER_RISING, + BRCM_AVS_HOST_INTR, priv); + if (ret) { + dev_err(dev, "IRQ request failed: %s (%d) -- %d\n", + BRCM_AVS_HOST_INTR, host_irq, ret); + goto unmap_intr_base; + } + + if (brcm_avs_is_firmware_loaded(priv)) + return 0; + + dev_err(dev, "AVS firmware is not loaded or doesn't support DVFS\n"); + ret = -ENODEV; + +unmap_intr_base: + iounmap(priv->avs_intr_base); +unmap_base: + iounmap(priv->base); + platform_set_drvdata(pdev, NULL); + + return ret; +} + +static int brcm_avs_cpufreq_init(struct cpufreq_policy *policy) +{ + struct cpufreq_frequency_table *freq_table; + struct platform_device *pdev; + struct private_data *priv; + struct device *dev; + int ret; + + pdev = cpufreq_get_driver_data(); + priv = platform_get_drvdata(pdev); + policy->driver_data = priv; + dev = &pdev->dev; + + freq_table = brcm_avs_get_freq_table(dev, priv); + if (IS_ERR(freq_table)) { + ret = PTR_ERR(freq_table); + dev_err(dev, "Couldn't determine frequency table (%d).\n", ret); + return ret; + } + + ret = cpufreq_table_validate_and_show(policy, freq_table); + if (ret) { + dev_err(dev, "invalid frequency table: %d\n", ret); + return ret; + } + + /* All cores share the same clock and thus the same policy. */ + cpumask_setall(policy->cpus); + + ret = __issue_avs_command(priv, AVS_CMD_ENABLE, false, NULL); + if (!ret) { + unsigned int pstate; + + ret = brcm_avs_get_pstate(priv, &pstate); + if (!ret) { + policy->cur = freq_table[pstate].frequency; + dev_info(dev, "registered\n"); + return 0; + } + } + + dev_err(dev, "couldn't initialize driver (%d)\n", ret); + + return ret; +} + +static ssize_t show_brcm_avs_pstate(struct cpufreq_policy *policy, char *buf) +{ + struct private_data *priv = policy->driver_data; + unsigned int pstate; + + if (brcm_avs_get_pstate(priv, &pstate)) + return sprintf(buf, "<unknown>\n"); + + return sprintf(buf, "%u\n", pstate); +} + +static ssize_t show_brcm_avs_mode(struct cpufreq_policy *policy, char *buf) +{ + struct private_data *priv = policy->driver_data; + struct pmap pmap; + + if (brcm_avs_get_pmap(priv, &pmap)) + return sprintf(buf, "<unknown>\n"); + + return sprintf(buf, "%s %u\n", brcm_avs_mode_to_string(pmap.mode), + pmap.mode); +} + +static ssize_t show_brcm_avs_pmap(struct cpufreq_policy *policy, char *buf) +{ + unsigned int mdiv_p0, mdiv_p1, mdiv_p2, mdiv_p3, mdiv_p4; + struct private_data *priv = policy->driver_data; + unsigned int ndiv, pdiv; + struct pmap pmap; + + if (brcm_avs_get_pmap(priv, &pmap)) + return sprintf(buf, "<unknown>\n"); + + brcm_avs_parse_p1(pmap.p1, &mdiv_p0, &pdiv, &ndiv); + brcm_avs_parse_p2(pmap.p2, &mdiv_p1, &mdiv_p2, &mdiv_p3, &mdiv_p4); + + return sprintf(buf, "0x%08x 0x%08x %u %u %u %u %u %u %u\n", + pmap.p1, pmap.p2, ndiv, pdiv, mdiv_p0, mdiv_p1, mdiv_p2, + mdiv_p3, mdiv_p4); +} + +static ssize_t show_brcm_avs_voltage(struct cpufreq_policy *policy, char *buf) +{ + struct private_data *priv = policy->driver_data; + + return sprintf(buf, "0x%08lx\n", brcm_avs_get_voltage(priv->base)); +} + +static ssize_t show_brcm_avs_frequency(struct cpufreq_policy *policy, char *buf) +{ + struct private_data *priv = policy->driver_data; + + return sprintf(buf, "0x%08lx\n", brcm_avs_get_frequency(priv->base)); +} + +cpufreq_freq_attr_ro(brcm_avs_pstate); +cpufreq_freq_attr_ro(brcm_avs_mode); +cpufreq_freq_attr_ro(brcm_avs_pmap); +cpufreq_freq_attr_ro(brcm_avs_voltage); +cpufreq_freq_attr_ro(brcm_avs_frequency); + +static struct freq_attr *brcm_avs_cpufreq_attr[] = { + &cpufreq_freq_attr_scaling_available_freqs, + &brcm_avs_pstate, + &brcm_avs_mode, + &brcm_avs_pmap, + &brcm_avs_voltage, + &brcm_avs_frequency, + NULL +}; + +static struct cpufreq_driver brcm_avs_driver = { + .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, + .verify = cpufreq_generic_frequency_table_verify, + .target_index = brcm_avs_target_index, + .get = brcm_avs_cpufreq_get, + .suspend = brcm_avs_suspend, + .resume = brcm_avs_resume, + .init = brcm_avs_cpufreq_init, + .attr = brcm_avs_cpufreq_attr, + .name = BRCM_AVS_CPUFREQ_PREFIX, +}; + +static int brcm_avs_cpufreq_probe(struct platform_device *pdev) +{ + int ret; + + ret = brcm_avs_prepare_init(pdev); + if (ret) + return ret; + + brcm_avs_driver.driver_data = pdev; + ret = cpufreq_register_driver(&brcm_avs_driver); + if (!ret) + brcm_avs_cpufreq_debug_init(pdev); + + return ret; +} + +static int brcm_avs_cpufreq_remove(struct platform_device *pdev) +{ + struct private_data *priv; + int ret; + + ret = cpufreq_unregister_driver(&brcm_avs_driver); + if (ret) + return ret; + + brcm_avs_cpufreq_debug_exit(pdev); + + priv = platform_get_drvdata(pdev); + iounmap(priv->base); + iounmap(priv->avs_intr_base); + platform_set_drvdata(pdev, NULL); + + return 0; +} + +static const struct of_device_id brcm_avs_cpufreq_match[] = { + { .compatible = BRCM_AVS_CPU_DATA }, + { } +}; +MODULE_DEVICE_TABLE(of, brcm_avs_cpufreq_match); + +static struct platform_driver brcm_avs_cpufreq_platdrv = { + .driver = { + .name = BRCM_AVS_CPUFREQ_NAME, + .of_match_table = brcm_avs_cpufreq_match, + }, + .probe = brcm_avs_cpufreq_probe, + .remove = brcm_avs_cpufreq_remove, +}; +module_platform_driver(brcm_avs_cpufreq_platdrv); + +MODULE_AUTHOR("Markus Mayer <mmayer@broadcom.com>"); +MODULE_DESCRIPTION("CPUfreq driver for Broadcom STB AVS"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index 4852d9efe74e..e82bb3c30b92 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -247,3 +247,10 @@ MODULE_DESCRIPTION("CPUFreq driver based on the ACPI CPPC v5.0+ spec"); MODULE_LICENSE("GPL"); late_initcall(cppc_cpufreq_init); + +static const struct acpi_device_id cppc_acpi_ids[] = { + {ACPI_PROCESSOR_DEVICE_HID, }, + {} +}; + +MODULE_DEVICE_TABLE(acpi, cppc_acpi_ids); diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index 71267626456b..bc97b6a4b1cf 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -26,6 +26,9 @@ static const struct of_device_id machines[] __initconst = { { .compatible = "allwinner,sun8i-a83t", }, { .compatible = "allwinner,sun8i-h3", }, + { .compatible = "arm,integrator-ap", }, + { .compatible = "arm,integrator-cp", }, + { .compatible = "hisilicon,hi6220", }, { .compatible = "fsl,imx27", }, @@ -34,6 +37,8 @@ static const struct of_device_id machines[] __initconst = { { .compatible = "fsl,imx7d", }, { .compatible = "marvell,berlin", }, + { .compatible = "marvell,pxa250", }, + { .compatible = "marvell,pxa270", }, { .compatible = "samsung,exynos3250", }, { .compatible = "samsung,exynos4210", }, @@ -50,6 +55,8 @@ static const struct of_device_id machines[] __initconst = { { .compatible = "renesas,r7s72100", }, { .compatible = "renesas,r8a73a4", }, { .compatible = "renesas,r8a7740", }, + { .compatible = "renesas,r8a7743", }, + { .compatible = "renesas,r8a7745", }, { .compatible = "renesas,r8a7778", }, { .compatible = "renesas,r8a7779", }, { .compatible = "renesas,r8a7790", }, @@ -72,6 +79,12 @@ static const struct of_device_id machines[] __initconst = { { .compatible = "sigma,tango4" }, + { .compatible = "socionext,uniphier-pro5", }, + { .compatible = "socionext,uniphier-pxs2", }, + { .compatible = "socionext,uniphier-ld6b", }, + { .compatible = "socionext,uniphier-ld11", }, + { .compatible = "socionext,uniphier-ld20", }, + { .compatible = "ti,am33xx", }, { .compatible = "ti,dra7", }, { .compatible = "ti,omap2", }, @@ -81,6 +94,8 @@ static const struct of_device_id machines[] __initconst = { { .compatible = "xlnx,zynq-7000", }, + { .compatible = "zte,zx296718", }, + { } }; diff --git a/drivers/cpufreq/cpufreq-dt.c b/drivers/cpufreq/cpufreq-dt.c index 5c07ae05d69a..269013311e79 100644 --- a/drivers/cpufreq/cpufreq-dt.c +++ b/drivers/cpufreq/cpufreq-dt.c @@ -28,6 +28,7 @@ #include "cpufreq-dt.h" struct private_data { + struct opp_table *opp_table; struct device *cpu_dev; struct thermal_cooling_device *cdev; const char *reg_name; @@ -143,6 +144,7 @@ static int resources_available(void) static int cpufreq_init(struct cpufreq_policy *policy) { struct cpufreq_frequency_table *freq_table; + struct opp_table *opp_table = NULL; struct private_data *priv; struct device *cpu_dev; struct clk *cpu_clk; @@ -186,8 +188,9 @@ static int cpufreq_init(struct cpufreq_policy *policy) */ name = find_supply_name(cpu_dev); if (name) { - ret = dev_pm_opp_set_regulator(cpu_dev, name); - if (ret) { + opp_table = dev_pm_opp_set_regulators(cpu_dev, &name, 1); + if (IS_ERR(opp_table)) { + ret = PTR_ERR(opp_table); dev_err(cpu_dev, "Failed to set regulator for cpu%d: %d\n", policy->cpu, ret); goto out_put_clk; @@ -237,6 +240,7 @@ static int cpufreq_init(struct cpufreq_policy *policy) } priv->reg_name = name; + priv->opp_table = opp_table; ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table); if (ret) { @@ -285,7 +289,7 @@ out_free_priv: out_free_opp: dev_pm_opp_of_cpumask_remove_table(policy->cpus); if (name) - dev_pm_opp_put_regulator(cpu_dev); + dev_pm_opp_put_regulators(opp_table); out_put_clk: clk_put(cpu_clk); @@ -300,7 +304,7 @@ static int cpufreq_exit(struct cpufreq_policy *policy) dev_pm_opp_free_cpufreq_table(priv->cpu_dev, &policy->freq_table); dev_pm_opp_of_cpumask_remove_table(policy->related_cpus); if (priv->reg_name) - dev_pm_opp_put_regulator(priv->cpu_dev); + dev_pm_opp_put_regulators(priv->opp_table); clk_put(policy->clk); kfree(priv); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 6e6c1fb60fbc..cc475eff90b3 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1526,7 +1526,10 @@ unsigned int cpufreq_get(unsigned int cpu) if (policy) { down_read(&policy->rwsem); - ret_freq = __cpufreq_get(policy); + + if (!policy_is_inactive(policy)) + ret_freq = __cpufreq_get(policy); + up_read(&policy->rwsem); cpufreq_cpu_put(policy); @@ -2254,17 +2257,19 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, * Useful for policy notifiers which have different necessities * at different times. */ -int cpufreq_update_policy(unsigned int cpu) +void cpufreq_update_policy(unsigned int cpu) { struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); struct cpufreq_policy new_policy; - int ret; if (!policy) - return -ENODEV; + return; down_write(&policy->rwsem); + if (policy_is_inactive(policy)) + goto unlock; + pr_debug("updating policy for CPU %u\n", cpu); memcpy(&new_policy, policy, sizeof(*policy)); new_policy.min = policy->user_policy.min; @@ -2275,24 +2280,20 @@ int cpufreq_update_policy(unsigned int cpu) * -> ask driver for current freq and notify governors about a change */ if (cpufreq_driver->get && !cpufreq_driver->setpolicy) { - if (cpufreq_suspended) { - ret = -EAGAIN; + if (cpufreq_suspended) goto unlock; - } + new_policy.cur = cpufreq_update_current_freq(policy); - if (WARN_ON(!new_policy.cur)) { - ret = -EIO; + if (WARN_ON(!new_policy.cur)) goto unlock; - } } - ret = cpufreq_set_policy(policy, &new_policy); + cpufreq_set_policy(policy, &new_policy); unlock: up_write(&policy->rwsem); cpufreq_cpu_put(policy); - return ret; } EXPORT_SYMBOL(cpufreq_update_policy); diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index 13475890d792..992f7c20760f 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -37,16 +37,16 @@ struct cs_dbs_tuners { #define DEF_SAMPLING_DOWN_FACTOR (1) #define MAX_SAMPLING_DOWN_FACTOR (10) -static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners, - struct cpufreq_policy *policy) +static inline unsigned int get_freq_step(struct cs_dbs_tuners *cs_tuners, + struct cpufreq_policy *policy) { - unsigned int freq_target = (cs_tuners->freq_step * policy->max) / 100; + unsigned int freq_step = (cs_tuners->freq_step * policy->max) / 100; /* max freq cannot be less than 100. But who knows... */ - if (unlikely(freq_target == 0)) - freq_target = DEF_FREQUENCY_STEP; + if (unlikely(freq_step == 0)) + freq_step = DEF_FREQUENCY_STEP; - return freq_target; + return freq_step; } /* @@ -55,10 +55,10 @@ static inline unsigned int get_freq_target(struct cs_dbs_tuners *cs_tuners, * sampling_down_factor, we check, if current idle time is more than 80% * (default), then we try to decrease frequency * - * Any frequency increase takes it to the maximum frequency. Frequency reduction - * happens at minimum steps of 5% (default) of maximum frequency + * Frequency updates happen at minimum steps of 5% (default) of maximum + * frequency */ -static unsigned int cs_dbs_timer(struct cpufreq_policy *policy) +static unsigned int cs_dbs_update(struct cpufreq_policy *policy) { struct policy_dbs_info *policy_dbs = policy->governor_data; struct cs_policy_dbs_info *dbs_info = to_dbs_info(policy_dbs); @@ -66,6 +66,7 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy) struct dbs_data *dbs_data = policy_dbs->dbs_data; struct cs_dbs_tuners *cs_tuners = dbs_data->tuners; unsigned int load = dbs_update(policy); + unsigned int freq_step; /* * break out if we 'cannot' reduce the speed as the user might @@ -82,6 +83,23 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy) if (requested_freq > policy->max || requested_freq < policy->min) requested_freq = policy->cur; + freq_step = get_freq_step(cs_tuners, policy); + + /* + * Decrease requested_freq one freq_step for each idle period that + * we didn't update the frequency. + */ + if (policy_dbs->idle_periods < UINT_MAX) { + unsigned int freq_steps = policy_dbs->idle_periods * freq_step; + + if (requested_freq > freq_steps) + requested_freq -= freq_steps; + else + requested_freq = policy->min; + + policy_dbs->idle_periods = UINT_MAX; + } + /* Check for frequency increase */ if (load > dbs_data->up_threshold) { dbs_info->down_skip = 0; @@ -90,7 +108,7 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy) if (requested_freq == policy->max) goto out; - requested_freq += get_freq_target(cs_tuners, policy); + requested_freq += freq_step; if (requested_freq > policy->max) requested_freq = policy->max; @@ -106,16 +124,14 @@ static unsigned int cs_dbs_timer(struct cpufreq_policy *policy) /* Check for frequency decrease */ if (load < cs_tuners->down_threshold) { - unsigned int freq_target; /* * if we cannot reduce the frequency anymore, break out early */ if (requested_freq == policy->min) goto out; - freq_target = get_freq_target(cs_tuners, policy); - if (requested_freq > freq_target) - requested_freq -= freq_target; + if (requested_freq > freq_step) + requested_freq -= freq_step; else requested_freq = policy->min; @@ -305,7 +321,7 @@ static void cs_start(struct cpufreq_policy *policy) static struct dbs_governor cs_governor = { .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("conservative"), .kobj_type = { .default_attrs = cs_attributes }, - .gov_dbs_timer = cs_dbs_timer, + .gov_dbs_update = cs_dbs_update, .alloc = cs_alloc, .free = cs_free, .init = cs_init, diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c index 642dd0f183a8..0196467280bd 100644 --- a/drivers/cpufreq/cpufreq_governor.c +++ b/drivers/cpufreq/cpufreq_governor.c @@ -61,7 +61,7 @@ ssize_t store_sampling_rate(struct gov_attr_set *attr_set, const char *buf, * entries can't be freed concurrently. */ list_for_each_entry(policy_dbs, &attr_set->policy_list, list) { - mutex_lock(&policy_dbs->timer_mutex); + mutex_lock(&policy_dbs->update_mutex); /* * On 32-bit architectures this may race with the * sample_delay_ns read in dbs_update_util_handler(), but that @@ -76,7 +76,7 @@ ssize_t store_sampling_rate(struct gov_attr_set *attr_set, const char *buf, * taken, so it shouldn't be significant. */ gov_update_sample_delay(policy_dbs, 0); - mutex_unlock(&policy_dbs->timer_mutex); + mutex_unlock(&policy_dbs->update_mutex); } return count; @@ -117,7 +117,7 @@ unsigned int dbs_update(struct cpufreq_policy *policy) struct policy_dbs_info *policy_dbs = policy->governor_data; struct dbs_data *dbs_data = policy_dbs->dbs_data; unsigned int ignore_nice = dbs_data->ignore_nice_load; - unsigned int max_load = 0; + unsigned int max_load = 0, idle_periods = UINT_MAX; unsigned int sampling_rate, io_busy, j; /* @@ -215,9 +215,19 @@ unsigned int dbs_update(struct cpufreq_policy *policy) j_cdbs->prev_load = load; } + if (time_elapsed > 2 * sampling_rate) { + unsigned int periods = time_elapsed / sampling_rate; + + if (periods < idle_periods) + idle_periods = periods; + } + if (load > max_load) max_load = load; } + + policy_dbs->idle_periods = idle_periods; + return max_load; } EXPORT_SYMBOL_GPL(dbs_update); @@ -236,9 +246,9 @@ static void dbs_work_handler(struct work_struct *work) * Make sure cpufreq_governor_limits() isn't evaluating load or the * ondemand governor isn't updating the sampling rate in parallel. */ - mutex_lock(&policy_dbs->timer_mutex); - gov_update_sample_delay(policy_dbs, gov->gov_dbs_timer(policy)); - mutex_unlock(&policy_dbs->timer_mutex); + mutex_lock(&policy_dbs->update_mutex); + gov_update_sample_delay(policy_dbs, gov->gov_dbs_update(policy)); + mutex_unlock(&policy_dbs->update_mutex); /* Allow the utilization update handler to queue up more work. */ atomic_set(&policy_dbs->work_count, 0); @@ -348,7 +358,7 @@ static struct policy_dbs_info *alloc_policy_dbs_info(struct cpufreq_policy *poli return NULL; policy_dbs->policy = policy; - mutex_init(&policy_dbs->timer_mutex); + mutex_init(&policy_dbs->update_mutex); atomic_set(&policy_dbs->work_count, 0); init_irq_work(&policy_dbs->irq_work, dbs_irq_work); INIT_WORK(&policy_dbs->work, dbs_work_handler); @@ -367,7 +377,7 @@ static void free_policy_dbs_info(struct policy_dbs_info *policy_dbs, { int j; - mutex_destroy(&policy_dbs->timer_mutex); + mutex_destroy(&policy_dbs->update_mutex); for_each_cpu(j, policy_dbs->policy->related_cpus) { struct cpu_dbs_info *j_cdbs = &per_cpu(cpu_dbs, j); @@ -547,10 +557,10 @@ void cpufreq_dbs_governor_limits(struct cpufreq_policy *policy) { struct policy_dbs_info *policy_dbs = policy->governor_data; - mutex_lock(&policy_dbs->timer_mutex); + mutex_lock(&policy_dbs->update_mutex); cpufreq_policy_apply_limits(policy); gov_update_sample_delay(policy_dbs, 0); - mutex_unlock(&policy_dbs->timer_mutex); + mutex_unlock(&policy_dbs->update_mutex); } EXPORT_SYMBOL_GPL(cpufreq_dbs_governor_limits); diff --git a/drivers/cpufreq/cpufreq_governor.h b/drivers/cpufreq/cpufreq_governor.h index ef1037e9c92b..f5717ca070cc 100644 --- a/drivers/cpufreq/cpufreq_governor.h +++ b/drivers/cpufreq/cpufreq_governor.h @@ -85,7 +85,7 @@ struct policy_dbs_info { * Per policy mutex that serializes load evaluation from limit-change * and work-handler. */ - struct mutex timer_mutex; + struct mutex update_mutex; u64 last_sample_time; s64 sample_delay_ns; @@ -97,6 +97,7 @@ struct policy_dbs_info { struct list_head list; /* Multiplier for increasing sample delay temporarily. */ unsigned int rate_mult; + unsigned int idle_periods; /* For conservative */ /* Status indicators */ bool is_shared; /* This object is used by multiple CPUs */ bool work_in_progress; /* Work is being queued up or in progress */ @@ -135,7 +136,7 @@ struct dbs_governor { */ struct dbs_data *gdbs_data; - unsigned int (*gov_dbs_timer)(struct cpufreq_policy *policy); + unsigned int (*gov_dbs_update)(struct cpufreq_policy *policy); struct policy_dbs_info *(*alloc)(void); void (*free)(struct policy_dbs_info *policy_dbs); int (*init)(struct dbs_data *dbs_data); diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index 3a1f49f5f4c6..4a017e895296 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -25,7 +25,7 @@ #define MAX_SAMPLING_DOWN_FACTOR (100000) #define MICRO_FREQUENCY_UP_THRESHOLD (95) #define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000) -#define MIN_FREQUENCY_UP_THRESHOLD (11) +#define MIN_FREQUENCY_UP_THRESHOLD (1) #define MAX_FREQUENCY_UP_THRESHOLD (100) static struct od_ops od_ops; @@ -169,7 +169,7 @@ static void od_update(struct cpufreq_policy *policy) } } -static unsigned int od_dbs_timer(struct cpufreq_policy *policy) +static unsigned int od_dbs_update(struct cpufreq_policy *policy) { struct policy_dbs_info *policy_dbs = policy->governor_data; struct dbs_data *dbs_data = policy_dbs->dbs_data; @@ -191,7 +191,7 @@ static unsigned int od_dbs_timer(struct cpufreq_policy *policy) od_update(policy); if (dbs_info->freq_lo) { - /* Setup timer for SUB_SAMPLE */ + /* Setup SUB_SAMPLE */ dbs_info->sample_type = OD_SUB_SAMPLE; return dbs_info->freq_hi_delay_us; } @@ -255,11 +255,11 @@ static ssize_t store_sampling_down_factor(struct gov_attr_set *attr_set, list_for_each_entry(policy_dbs, &attr_set->policy_list, list) { /* * Doing this without locking might lead to using different - * rate_mult values in od_update() and od_dbs_timer(). + * rate_mult values in od_update() and od_dbs_update(). */ - mutex_lock(&policy_dbs->timer_mutex); + mutex_lock(&policy_dbs->update_mutex); policy_dbs->rate_mult = 1; - mutex_unlock(&policy_dbs->timer_mutex); + mutex_unlock(&policy_dbs->update_mutex); } return count; @@ -374,8 +374,7 @@ static int od_init(struct dbs_data *dbs_data) dbs_data->up_threshold = MICRO_FREQUENCY_UP_THRESHOLD; /* * In nohz/micro accounting case we set the minimum frequency - * not depending on HZ, but fixed (very low). The deferred - * timer might skip some samples if idle/sleeping as needed. + * not depending on HZ, but fixed (very low). */ dbs_data->min_sampling_rate = MICRO_FREQUENCY_MIN_SAMPLE_RATE; } else { @@ -415,7 +414,7 @@ static struct od_ops od_ops = { static struct dbs_governor od_dbs_gov = { .gov = CPUFREQ_DBS_GOVERNOR_INITIALIZER("ondemand"), .kobj_type = { .default_attrs = od_attributes }, - .gov_dbs_timer = od_dbs_timer, + .gov_dbs_update = od_dbs_update, .alloc = od_alloc, .free = od_free, .init = od_init, diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c index 06d3abdffd3a..ac284e66839c 100644 --- a/drivers/cpufreq/cpufreq_stats.c +++ b/drivers/cpufreq/cpufreq_stats.c @@ -41,6 +41,18 @@ static int cpufreq_stats_update(struct cpufreq_stats *stats) return 0; } +static void cpufreq_stats_clear_table(struct cpufreq_stats *stats) +{ + unsigned int count = stats->max_state; + + memset(stats->time_in_state, 0, count * sizeof(u64)); +#ifdef CONFIG_CPU_FREQ_STAT_DETAILS + memset(stats->trans_table, 0, count * count * sizeof(int)); +#endif + stats->last_time = get_jiffies_64(); + stats->total_trans = 0; +} + static ssize_t show_total_trans(struct cpufreq_policy *policy, char *buf) { return sprintf(buf, "%d\n", policy->stats->total_trans); @@ -64,6 +76,14 @@ static ssize_t show_time_in_state(struct cpufreq_policy *policy, char *buf) return len; } +static ssize_t store_reset(struct cpufreq_policy *policy, const char *buf, + size_t count) +{ + /* We don't care what is written to the attribute. */ + cpufreq_stats_clear_table(policy->stats); + return count; +} + #ifdef CONFIG_CPU_FREQ_STAT_DETAILS static ssize_t show_trans_table(struct cpufreq_policy *policy, char *buf) { @@ -113,10 +133,12 @@ cpufreq_freq_attr_ro(trans_table); cpufreq_freq_attr_ro(total_trans); cpufreq_freq_attr_ro(time_in_state); +cpufreq_freq_attr_wo(reset); static struct attribute *default_attrs[] = { &total_trans.attr, &time_in_state.attr, + &reset.attr, #ifdef CONFIG_CPU_FREQ_STAT_DETAILS &trans_table.attr, #endif diff --git a/drivers/cpufreq/integrator-cpufreq.c b/drivers/cpufreq/integrator-cpufreq.c deleted file mode 100644 index 79e3ff2771a6..000000000000 --- a/drivers/cpufreq/integrator-cpufreq.c +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright (C) 2001-2002 Deep Blue Solutions Ltd. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * - * CPU support functions - */ -#include <linux/module.h> -#include <linux/types.h> -#include <linux/kernel.h> -#include <linux/cpufreq.h> -#include <linux/sched.h> -#include <linux/smp.h> -#include <linux/init.h> -#include <linux/io.h> -#include <linux/platform_device.h> -#include <linux/of.h> -#include <linux/of_address.h> - -#include <asm/mach-types.h> -#include <asm/hardware/icst.h> - -static void __iomem *cm_base; -/* The cpufreq driver only use the OSC register */ -#define INTEGRATOR_HDR_OSC_OFFSET 0x08 -#define INTEGRATOR_HDR_LOCK_OFFSET 0x14 - -static struct cpufreq_driver integrator_driver; - -static const struct icst_params lclk_params = { - .ref = 24000000, - .vco_max = ICST525_VCO_MAX_5V, - .vco_min = ICST525_VCO_MIN, - .vd_min = 8, - .vd_max = 132, - .rd_min = 24, - .rd_max = 24, - .s2div = icst525_s2div, - .idx2s = icst525_idx2s, -}; - -static const struct icst_params cclk_params = { - .ref = 24000000, - .vco_max = ICST525_VCO_MAX_5V, - .vco_min = ICST525_VCO_MIN, - .vd_min = 12, - .vd_max = 160, - .rd_min = 24, - .rd_max = 24, - .s2div = icst525_s2div, - .idx2s = icst525_idx2s, -}; - -/* - * Validate the speed policy. - */ -static int integrator_verify_policy(struct cpufreq_policy *policy) -{ - struct icst_vco vco; - - cpufreq_verify_within_cpu_limits(policy); - - vco = icst_hz_to_vco(&cclk_params, policy->max * 1000); - policy->max = icst_hz(&cclk_params, vco) / 1000; - - vco = icst_hz_to_vco(&cclk_params, policy->min * 1000); - policy->min = icst_hz(&cclk_params, vco) / 1000; - - cpufreq_verify_within_cpu_limits(policy); - return 0; -} - - -static int integrator_set_target(struct cpufreq_policy *policy, - unsigned int target_freq, - unsigned int relation) -{ - cpumask_t cpus_allowed; - int cpu = policy->cpu; - struct icst_vco vco; - struct cpufreq_freqs freqs; - u_int cm_osc; - - /* - * Save this threads cpus_allowed mask. - */ - cpus_allowed = current->cpus_allowed; - - /* - * Bind to the specified CPU. When this call returns, - * we should be running on the right CPU. - */ - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - BUG_ON(cpu != smp_processor_id()); - - /* get current setting */ - cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET); - - if (machine_is_integrator()) - vco.s = (cm_osc >> 8) & 7; - else if (machine_is_cintegrator()) - vco.s = 1; - vco.v = cm_osc & 255; - vco.r = 22; - freqs.old = icst_hz(&cclk_params, vco) / 1000; - - /* icst_hz_to_vco rounds down -- so we need the next - * larger freq in case of CPUFREQ_RELATION_L. - */ - if (relation == CPUFREQ_RELATION_L) - target_freq += 999; - if (target_freq > policy->max) - target_freq = policy->max; - vco = icst_hz_to_vco(&cclk_params, target_freq * 1000); - freqs.new = icst_hz(&cclk_params, vco) / 1000; - - if (freqs.old == freqs.new) { - set_cpus_allowed_ptr(current, &cpus_allowed); - return 0; - } - - cpufreq_freq_transition_begin(policy, &freqs); - - cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET); - - if (machine_is_integrator()) { - cm_osc &= 0xfffff800; - cm_osc |= vco.s << 8; - } else if (machine_is_cintegrator()) { - cm_osc &= 0xffffff00; - } - cm_osc |= vco.v; - - __raw_writel(0xa05f, cm_base + INTEGRATOR_HDR_LOCK_OFFSET); - __raw_writel(cm_osc, cm_base + INTEGRATOR_HDR_OSC_OFFSET); - __raw_writel(0, cm_base + INTEGRATOR_HDR_LOCK_OFFSET); - - /* - * Restore the CPUs allowed mask. - */ - set_cpus_allowed_ptr(current, &cpus_allowed); - - cpufreq_freq_transition_end(policy, &freqs, 0); - - return 0; -} - -static unsigned int integrator_get(unsigned int cpu) -{ - cpumask_t cpus_allowed; - unsigned int current_freq; - u_int cm_osc; - struct icst_vco vco; - - cpus_allowed = current->cpus_allowed; - - set_cpus_allowed_ptr(current, cpumask_of(cpu)); - BUG_ON(cpu != smp_processor_id()); - - /* detect memory etc. */ - cm_osc = __raw_readl(cm_base + INTEGRATOR_HDR_OSC_OFFSET); - - if (machine_is_integrator()) - vco.s = (cm_osc >> 8) & 7; - else - vco.s = 1; - vco.v = cm_osc & 255; - vco.r = 22; - - current_freq = icst_hz(&cclk_params, vco) / 1000; /* current freq */ - - set_cpus_allowed_ptr(current, &cpus_allowed); - - return current_freq; -} - -static int integrator_cpufreq_init(struct cpufreq_policy *policy) -{ - - /* set default policy and cpuinfo */ - policy->max = policy->cpuinfo.max_freq = 160000; - policy->min = policy->cpuinfo.min_freq = 12000; - policy->cpuinfo.transition_latency = 1000000; /* 1 ms, assumed */ - - return 0; -} - -static struct cpufreq_driver integrator_driver = { - .flags = CPUFREQ_NEED_INITIAL_FREQ_CHECK, - .verify = integrator_verify_policy, - .target = integrator_set_target, - .get = integrator_get, - .init = integrator_cpufreq_init, - .name = "integrator", -}; - -static int __init integrator_cpufreq_probe(struct platform_device *pdev) -{ - struct resource *res; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - - cm_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); - if (!cm_base) - return -ENODEV; - - return cpufreq_register_driver(&integrator_driver); -} - -static int __exit integrator_cpufreq_remove(struct platform_device *pdev) -{ - return cpufreq_unregister_driver(&integrator_driver); -} - -static const struct of_device_id integrator_cpufreq_match[] = { - { .compatible = "arm,core-module-integrator"}, - { }, -}; - -MODULE_DEVICE_TABLE(of, integrator_cpufreq_match); - -static struct platform_driver integrator_cpufreq_driver = { - .driver = { - .name = "integrator-cpufreq", - .of_match_table = integrator_cpufreq_match, - }, - .remove = __exit_p(integrator_cpufreq_remove), -}; - -module_platform_driver_probe(integrator_cpufreq_driver, - integrator_cpufreq_probe); - -MODULE_AUTHOR("Russell M. King"); -MODULE_DESCRIPTION("cpufreq driver for ARM Integrator CPUs"); -MODULE_LICENSE("GPL"); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index e8dc42fc0915..6acbd4af632e 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -37,6 +37,8 @@ #include <asm/cpufeature.h> #include <asm/intel-family.h> +#define INTEL_CPUFREQ_TRANSITION_LATENCY 20000 + #define ATOM_RATIOS 0x66a #define ATOM_VIDS 0x66b #define ATOM_TURBO_RATIOS 0x66c @@ -53,6 +55,8 @@ #define EXT_BITS 6 #define EXT_FRAC_BITS (EXT_BITS + FRAC_BITS) +#define fp_ext_toint(X) ((X) >> EXT_FRAC_BITS) +#define int_ext_tofp(X) ((int64_t)(X) << EXT_FRAC_BITS) static inline int32_t mul_fp(int32_t x, int32_t y) { @@ -123,6 +127,8 @@ struct sample { * @scaling: Scaling factor to convert frequency to cpufreq * frequency units * @turbo_pstate: Max Turbo P state possible for this platform + * @max_freq: @max_pstate frequency in cpufreq units + * @turbo_freq: @turbo_pstate frequency in cpufreq units * * Stores the per cpu model P state limits and current P state. */ @@ -133,6 +139,8 @@ struct pstate_data { int max_pstate_physical; int scaling; int turbo_pstate; + unsigned int max_freq; + unsigned int turbo_freq; }; /** @@ -178,6 +186,48 @@ struct _pid { }; /** + * struct perf_limits - Store user and policy limits + * @no_turbo: User requested turbo state from intel_pstate sysfs + * @turbo_disabled: Platform turbo status either from msr + * MSR_IA32_MISC_ENABLE or when maximum available pstate + * matches the maximum turbo pstate + * @max_perf_pct: Effective maximum performance limit in percentage, this + * is minimum of either limits enforced by cpufreq policy + * or limits from user set limits via intel_pstate sysfs + * @min_perf_pct: Effective minimum performance limit in percentage, this + * is maximum of either limits enforced by cpufreq policy + * or limits from user set limits via intel_pstate sysfs + * @max_perf: This is a scaled value between 0 to 255 for max_perf_pct + * This value is used to limit max pstate + * @min_perf: This is a scaled value between 0 to 255 for min_perf_pct + * This value is used to limit min pstate + * @max_policy_pct: The maximum performance in percentage enforced by + * cpufreq setpolicy interface + * @max_sysfs_pct: The maximum performance in percentage enforced by + * intel pstate sysfs interface, unused when per cpu + * controls are enforced + * @min_policy_pct: The minimum performance in percentage enforced by + * cpufreq setpolicy interface + * @min_sysfs_pct: The minimum performance in percentage enforced by + * intel pstate sysfs interface, unused when per cpu + * controls are enforced + * + * Storage for user and policy defined limits. + */ +struct perf_limits { + int no_turbo; + int turbo_disabled; + int max_perf_pct; + int min_perf_pct; + int32_t max_perf; + int32_t min_perf; + int max_policy_pct; + int max_sysfs_pct; + int min_policy_pct; + int min_sysfs_pct; +}; + +/** * struct cpudata - Per CPU instance data storage * @cpu: CPU number for this instance data * @policy: CPUFreq policy value @@ -195,8 +245,19 @@ struct _pid { * @prev_cummulative_iowait: IO Wait time difference from last and * current sample * @sample: Storage for storing last Sample data + * @perf_limits: Pointer to perf_limit unique to this CPU + * Not all field in the structure are applicable + * when per cpu controls are enforced * @acpi_perf_data: Stores ACPI perf information read from _PSS * @valid_pss_table: Set to true for valid ACPI _PSS entries found + * @epp_powersave: Last saved HWP energy performance preference + * (EPP) or energy performance bias (EPB), + * when policy switched to performance + * @epp_policy: Last saved policy used to set EPP/EPB + * @epp_default: Power on default HWP energy performance + * preference/bias + * @epp_saved: Saved EPP/EPB during system suspend or CPU offline + * operation * * This structure stores per CPU instance data for all CPUs. */ @@ -218,11 +279,16 @@ struct cpudata { u64 prev_tsc; u64 prev_cummulative_iowait; struct sample sample; + struct perf_limits *perf_limits; #ifdef CONFIG_ACPI struct acpi_processor_performance acpi_perf_data; bool valid_pss_table; #endif unsigned int iowait_boost; + s16 epp_powersave; + s16 epp_policy; + s16 epp_default; + s16 epp_saved; }; static struct cpudata **all_cpu_data; @@ -236,7 +302,6 @@ static struct cpudata **all_cpu_data; * @p_gain_pct: PID proportional gain * @i_gain_pct: PID integral gain * @d_gain_pct: PID derivative gain - * @boost_iowait: Whether or not to use iowait boosting. * * Stores per CPU model static PID configuration data. */ @@ -248,7 +313,6 @@ struct pstate_adjust_policy { int p_gain_pct; int d_gain_pct; int i_gain_pct; - bool boost_iowait; }; /** @@ -292,58 +356,19 @@ static inline int32_t get_target_pstate_use_cpu_load(struct cpudata *cpu); static struct pstate_adjust_policy pid_params __read_mostly; static struct pstate_funcs pstate_funcs __read_mostly; static int hwp_active __read_mostly; +static bool per_cpu_limits __read_mostly; #ifdef CONFIG_ACPI static bool acpi_ppc; #endif -/** - * struct perf_limits - Store user and policy limits - * @no_turbo: User requested turbo state from intel_pstate sysfs - * @turbo_disabled: Platform turbo status either from msr - * MSR_IA32_MISC_ENABLE or when maximum available pstate - * matches the maximum turbo pstate - * @max_perf_pct: Effective maximum performance limit in percentage, this - * is minimum of either limits enforced by cpufreq policy - * or limits from user set limits via intel_pstate sysfs - * @min_perf_pct: Effective minimum performance limit in percentage, this - * is maximum of either limits enforced by cpufreq policy - * or limits from user set limits via intel_pstate sysfs - * @max_perf: This is a scaled value between 0 to 255 for max_perf_pct - * This value is used to limit max pstate - * @min_perf: This is a scaled value between 0 to 255 for min_perf_pct - * This value is used to limit min pstate - * @max_policy_pct: The maximum performance in percentage enforced by - * cpufreq setpolicy interface - * @max_sysfs_pct: The maximum performance in percentage enforced by - * intel pstate sysfs interface - * @min_policy_pct: The minimum performance in percentage enforced by - * cpufreq setpolicy interface - * @min_sysfs_pct: The minimum performance in percentage enforced by - * intel pstate sysfs interface - * - * Storage for user and policy defined limits. - */ -struct perf_limits { - int no_turbo; - int turbo_disabled; - int max_perf_pct; - int min_perf_pct; - int32_t max_perf; - int32_t min_perf; - int max_policy_pct; - int max_sysfs_pct; - int min_policy_pct; - int min_sysfs_pct; -}; - static struct perf_limits performance_limits = { .no_turbo = 0, .turbo_disabled = 0, .max_perf_pct = 100, - .max_perf = int_tofp(1), + .max_perf = int_ext_tofp(1), .min_perf_pct = 100, - .min_perf = int_tofp(1), + .min_perf = int_ext_tofp(1), .max_policy_pct = 100, .max_sysfs_pct = 100, .min_policy_pct = 0, @@ -354,7 +379,7 @@ static struct perf_limits powersave_limits = { .no_turbo = 0, .turbo_disabled = 0, .max_perf_pct = 100, - .max_perf = int_tofp(1), + .max_perf = int_ext_tofp(1), .min_perf_pct = 0, .min_perf = 0, .max_policy_pct = 100, @@ -369,6 +394,8 @@ static struct perf_limits *limits = &performance_limits; static struct perf_limits *limits = &powersave_limits; #endif +static DEFINE_MUTEX(intel_pstate_limits_lock); + #ifdef CONFIG_ACPI static bool intel_pstate_get_ppc_enable_status(void) @@ -513,11 +540,11 @@ static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) } #else -static void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) +static inline void intel_pstate_init_acpi_perf_limits(struct cpufreq_policy *policy) { } -static void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) +static inline void intel_pstate_exit_perf_limits(struct cpufreq_policy *policy) { } #endif @@ -613,24 +640,252 @@ static inline void update_turbo_state(void) cpu->pstate.max_pstate == cpu->pstate.turbo_pstate); } +static s16 intel_pstate_get_epb(struct cpudata *cpu_data) +{ + u64 epb; + int ret; + + if (!static_cpu_has(X86_FEATURE_EPB)) + return -ENXIO; + + ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); + if (ret) + return (s16)ret; + + return (s16)(epb & 0x0f); +} + +static s16 intel_pstate_get_epp(struct cpudata *cpu_data, u64 hwp_req_data) +{ + s16 epp; + + if (static_cpu_has(X86_FEATURE_HWP_EPP)) { + /* + * When hwp_req_data is 0, means that caller didn't read + * MSR_HWP_REQUEST, so need to read and get EPP. + */ + if (!hwp_req_data) { + epp = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, + &hwp_req_data); + if (epp) + return epp; + } + epp = (hwp_req_data >> 24) & 0xff; + } else { + /* When there is no EPP present, HWP uses EPB settings */ + epp = intel_pstate_get_epb(cpu_data); + } + + return epp; +} + +static int intel_pstate_set_epb(int cpu, s16 pref) +{ + u64 epb; + int ret; + + if (!static_cpu_has(X86_FEATURE_EPB)) + return -ENXIO; + + ret = rdmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, &epb); + if (ret) + return ret; + + epb = (epb & ~0x0f) | pref; + wrmsrl_on_cpu(cpu, MSR_IA32_ENERGY_PERF_BIAS, epb); + + return 0; +} + +/* + * EPP/EPB display strings corresponding to EPP index in the + * energy_perf_strings[] + * index String + *------------------------------------- + * 0 default + * 1 performance + * 2 balance_performance + * 3 balance_power + * 4 power + */ +static const char * const energy_perf_strings[] = { + "default", + "performance", + "balance_performance", + "balance_power", + "power", + NULL +}; + +static int intel_pstate_get_energy_pref_index(struct cpudata *cpu_data) +{ + s16 epp; + int index = -EINVAL; + + epp = intel_pstate_get_epp(cpu_data, 0); + if (epp < 0) + return epp; + + if (static_cpu_has(X86_FEATURE_HWP_EPP)) { + /* + * Range: + * 0x00-0x3F : Performance + * 0x40-0x7F : Balance performance + * 0x80-0xBF : Balance power + * 0xC0-0xFF : Power + * The EPP is a 8 bit value, but our ranges restrict the + * value which can be set. Here only using top two bits + * effectively. + */ + index = (epp >> 6) + 1; + } else if (static_cpu_has(X86_FEATURE_EPB)) { + /* + * Range: + * 0x00-0x03 : Performance + * 0x04-0x07 : Balance performance + * 0x08-0x0B : Balance power + * 0x0C-0x0F : Power + * The EPB is a 4 bit value, but our ranges restrict the + * value which can be set. Here only using top two bits + * effectively. + */ + index = (epp >> 2) + 1; + } + + return index; +} + +static int intel_pstate_set_energy_pref_index(struct cpudata *cpu_data, + int pref_index) +{ + int epp = -EINVAL; + int ret; + + if (!pref_index) + epp = cpu_data->epp_default; + + mutex_lock(&intel_pstate_limits_lock); + + if (static_cpu_has(X86_FEATURE_HWP_EPP)) { + u64 value; + + ret = rdmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, &value); + if (ret) + goto return_pref; + + value &= ~GENMASK_ULL(31, 24); + + /* + * If epp is not default, convert from index into + * energy_perf_strings to epp value, by shifting 6 + * bits left to use only top two bits in epp. + * The resultant epp need to shifted by 24 bits to + * epp position in MSR_HWP_REQUEST. + */ + if (epp == -EINVAL) + epp = (pref_index - 1) << 6; + + value |= (u64)epp << 24; + ret = wrmsrl_on_cpu(cpu_data->cpu, MSR_HWP_REQUEST, value); + } else { + if (epp == -EINVAL) + epp = (pref_index - 1) << 2; + ret = intel_pstate_set_epb(cpu_data->cpu, epp); + } +return_pref: + mutex_unlock(&intel_pstate_limits_lock); + + return ret; +} + +static ssize_t show_energy_performance_available_preferences( + struct cpufreq_policy *policy, char *buf) +{ + int i = 0; + int ret = 0; + + while (energy_perf_strings[i] != NULL) + ret += sprintf(&buf[ret], "%s ", energy_perf_strings[i++]); + + ret += sprintf(&buf[ret], "\n"); + + return ret; +} + +cpufreq_freq_attr_ro(energy_performance_available_preferences); + +static ssize_t store_energy_performance_preference( + struct cpufreq_policy *policy, const char *buf, size_t count) +{ + struct cpudata *cpu_data = all_cpu_data[policy->cpu]; + char str_preference[21]; + int ret, i = 0; + + ret = sscanf(buf, "%20s", str_preference); + if (ret != 1) + return -EINVAL; + + while (energy_perf_strings[i] != NULL) { + if (!strcmp(str_preference, energy_perf_strings[i])) { + intel_pstate_set_energy_pref_index(cpu_data, i); + return count; + } + ++i; + } + + return -EINVAL; +} + +static ssize_t show_energy_performance_preference( + struct cpufreq_policy *policy, char *buf) +{ + struct cpudata *cpu_data = all_cpu_data[policy->cpu]; + int preference; + + preference = intel_pstate_get_energy_pref_index(cpu_data); + if (preference < 0) + return preference; + + return sprintf(buf, "%s\n", energy_perf_strings[preference]); +} + +cpufreq_freq_attr_rw(energy_performance_preference); + +static struct freq_attr *hwp_cpufreq_attrs[] = { + &energy_performance_preference, + &energy_performance_available_preferences, + NULL, +}; + static void intel_pstate_hwp_set(const struct cpumask *cpumask) { int min, hw_min, max, hw_max, cpu, range, adj_range; + struct perf_limits *perf_limits = limits; u64 value, cap; for_each_cpu(cpu, cpumask) { + int max_perf_pct, min_perf_pct; + struct cpudata *cpu_data = all_cpu_data[cpu]; + s16 epp; + + if (per_cpu_limits) + perf_limits = all_cpu_data[cpu]->perf_limits; + rdmsrl_on_cpu(cpu, MSR_HWP_CAPABILITIES, &cap); hw_min = HWP_LOWEST_PERF(cap); hw_max = HWP_HIGHEST_PERF(cap); range = hw_max - hw_min; + max_perf_pct = perf_limits->max_perf_pct; + min_perf_pct = perf_limits->min_perf_pct; + rdmsrl_on_cpu(cpu, MSR_HWP_REQUEST, &value); - adj_range = limits->min_perf_pct * range / 100; + adj_range = min_perf_pct * range / 100; min = hw_min + adj_range; value &= ~HWP_MIN_PERF(~0L); value |= HWP_MIN_PERF(min); - adj_range = limits->max_perf_pct * range / 100; + adj_range = max_perf_pct * range / 100; max = hw_min + adj_range; if (limits->no_turbo) { hw_max = HWP_GUARANTEED_PERF(cap); @@ -640,6 +895,53 @@ static void intel_pstate_hwp_set(const struct cpumask *cpumask) value &= ~HWP_MAX_PERF(~0L); value |= HWP_MAX_PERF(max); + + if (cpu_data->epp_policy == cpu_data->policy) + goto skip_epp; + + cpu_data->epp_policy = cpu_data->policy; + + if (cpu_data->epp_saved >= 0) { + epp = cpu_data->epp_saved; + cpu_data->epp_saved = -EINVAL; + goto update_epp; + } + + if (cpu_data->policy == CPUFREQ_POLICY_PERFORMANCE) { + epp = intel_pstate_get_epp(cpu_data, value); + cpu_data->epp_powersave = epp; + /* If EPP read was failed, then don't try to write */ + if (epp < 0) + goto skip_epp; + + + epp = 0; + } else { + /* skip setting EPP, when saved value is invalid */ + if (cpu_data->epp_powersave < 0) + goto skip_epp; + + /* + * No need to restore EPP when it is not zero. This + * means: + * - Policy is not changed + * - user has manually changed + * - Error reading EPB + */ + epp = intel_pstate_get_epp(cpu_data, value); + if (epp) + goto skip_epp; + + epp = cpu_data->epp_powersave; + } +update_epp: + if (static_cpu_has(X86_FEATURE_HWP_EPP)) { + value &= ~GENMASK_ULL(31, 24); + value |= (u64)epp << 24; + } else { + intel_pstate_set_epb(cpu, epp); + } +skip_epp: wrmsrl_on_cpu(cpu, MSR_HWP_REQUEST, value); } } @@ -652,6 +954,28 @@ static int intel_pstate_hwp_set_policy(struct cpufreq_policy *policy) return 0; } +static int intel_pstate_hwp_save_state(struct cpufreq_policy *policy) +{ + struct cpudata *cpu_data = all_cpu_data[policy->cpu]; + + if (!hwp_active) + return 0; + + cpu_data->epp_saved = intel_pstate_get_epp(cpu_data, 0); + + return 0; +} + +static int intel_pstate_resume(struct cpufreq_policy *policy) +{ + if (!hwp_active) + return 0; + + all_cpu_data[policy->cpu]->epp_policy = 0; + + return intel_pstate_hwp_set_policy(policy); +} + static void intel_pstate_hwp_set_online_cpus(void) { get_online_cpus(); @@ -694,8 +1018,10 @@ static void __init intel_pstate_debug_expose_params(void) struct dentry *debugfs_parent; int i = 0; - if (hwp_active) + if (hwp_active || + pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load) return; + debugfs_parent = debugfs_create_dir("pstate_snb", NULL); if (IS_ERR_OR_NULL(debugfs_parent)) return; @@ -768,9 +1094,12 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, if (ret != 1) return -EINVAL; + mutex_lock(&intel_pstate_limits_lock); + update_turbo_state(); if (limits->turbo_disabled) { pr_warn("Turbo disabled by BIOS or unavailable on processor\n"); + mutex_unlock(&intel_pstate_limits_lock); return -EPERM; } @@ -779,6 +1108,8 @@ static ssize_t store_no_turbo(struct kobject *a, struct attribute *b, if (hwp_active) intel_pstate_hwp_set_online_cpus(); + mutex_unlock(&intel_pstate_limits_lock); + return count; } @@ -792,6 +1123,8 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, if (ret != 1) return -EINVAL; + mutex_lock(&intel_pstate_limits_lock); + limits->max_sysfs_pct = clamp_t(int, input, 0 , 100); limits->max_perf_pct = min(limits->max_policy_pct, limits->max_sysfs_pct); @@ -799,10 +1132,13 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct attribute *b, limits->max_perf_pct); limits->max_perf_pct = max(limits->min_perf_pct, limits->max_perf_pct); - limits->max_perf = div_fp(limits->max_perf_pct, 100); + limits->max_perf = div_ext_fp(limits->max_perf_pct, 100); if (hwp_active) intel_pstate_hwp_set_online_cpus(); + + mutex_unlock(&intel_pstate_limits_lock); + return count; } @@ -816,6 +1152,8 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, if (ret != 1) return -EINVAL; + mutex_lock(&intel_pstate_limits_lock); + limits->min_sysfs_pct = clamp_t(int, input, 0 , 100); limits->min_perf_pct = max(limits->min_policy_pct, limits->min_sysfs_pct); @@ -823,10 +1161,13 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct attribute *b, limits->min_perf_pct); limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct); - limits->min_perf = div_fp(limits->min_perf_pct, 100); + limits->min_perf = div_ext_fp(limits->min_perf_pct, 100); if (hwp_active) intel_pstate_hwp_set_online_cpus(); + + mutex_unlock(&intel_pstate_limits_lock); + return count; } @@ -841,8 +1182,6 @@ define_one_global_ro(num_pstates); static struct attribute *intel_pstate_attributes[] = { &no_turbo.attr, - &max_perf_pct.attr, - &min_perf_pct.attr, &turbo_pct.attr, &num_pstates.attr, NULL @@ -859,9 +1198,26 @@ static void __init intel_pstate_sysfs_expose_params(void) intel_pstate_kobject = kobject_create_and_add("intel_pstate", &cpu_subsys.dev_root->kobj); - BUG_ON(!intel_pstate_kobject); + if (WARN_ON(!intel_pstate_kobject)) + return; + rc = sysfs_create_group(intel_pstate_kobject, &intel_pstate_attr_group); - BUG_ON(rc); + if (WARN_ON(rc)) + return; + + /* + * If per cpu limits are enforced there are no global limits, so + * return without creating max/min_perf_pct attributes + */ + if (per_cpu_limits) + return; + + rc = sysfs_create_file(intel_pstate_kobject, &max_perf_pct.attr); + WARN_ON(rc); + + rc = sysfs_create_file(intel_pstate_kobject, &min_perf_pct.attr); + WARN_ON(rc); + } /************************** sysfs end ************************/ @@ -872,6 +1228,9 @@ static void intel_pstate_hwp_enable(struct cpudata *cpudata) wrmsrl_on_cpu(cpudata->cpu, MSR_HWP_INTERRUPT, 0x00); wrmsrl_on_cpu(cpudata->cpu, MSR_PM_ENABLE, 0x1); + cpudata->epp_policy = 0; + if (cpudata->epp_default == -EINVAL) + cpudata->epp_default = intel_pstate_get_epp(cpudata, 0); } static int atom_get_min_pstate(void) @@ -1099,7 +1458,6 @@ static const struct cpu_defaults silvermont_params = { .p_gain_pct = 14, .d_gain_pct = 0, .i_gain_pct = 4, - .boost_iowait = true, }, .funcs = { .get_max = atom_get_max_pstate, @@ -1121,7 +1479,6 @@ static const struct cpu_defaults airmont_params = { .p_gain_pct = 14, .d_gain_pct = 0, .i_gain_pct = 4, - .boost_iowait = true, }, .funcs = { .get_max = atom_get_max_pstate, @@ -1163,7 +1520,6 @@ static const struct cpu_defaults bxt_params = { .p_gain_pct = 14, .d_gain_pct = 0, .i_gain_pct = 4, - .boost_iowait = true, }, .funcs = { .get_max = core_get_max_pstate, @@ -1181,20 +1537,24 @@ static void intel_pstate_get_min_max(struct cpudata *cpu, int *min, int *max) int max_perf = cpu->pstate.turbo_pstate; int max_perf_adj; int min_perf; + struct perf_limits *perf_limits = limits; if (limits->no_turbo || limits->turbo_disabled) max_perf = cpu->pstate.max_pstate; + if (per_cpu_limits) + perf_limits = cpu->perf_limits; + /* * performance can be limited by user through sysfs, by cpufreq * policy, or by cpu specific default values determined through * experimentation. */ - max_perf_adj = fp_toint(max_perf * limits->max_perf); + max_perf_adj = fp_ext_toint(max_perf * perf_limits->max_perf); *max = clamp_t(int, max_perf_adj, cpu->pstate.min_pstate, cpu->pstate.turbo_pstate); - min_perf = fp_toint(max_perf * limits->min_perf); + min_perf = fp_ext_toint(max_perf * perf_limits->min_perf); *min = clamp_t(int, min_perf, cpu->pstate.min_pstate, max_perf); } @@ -1232,6 +1592,8 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(); cpu->pstate.turbo_pstate = pstate_funcs.get_turbo(); cpu->pstate.scaling = pstate_funcs.get_scaling(); + cpu->pstate.max_freq = cpu->pstate.max_pstate * cpu->pstate.scaling; + cpu->pstate.turbo_freq = cpu->pstate.turbo_pstate * cpu->pstate.scaling; if (pstate_funcs.get_vid) pstate_funcs.get_vid(cpu); @@ -1370,15 +1732,19 @@ static inline int32_t get_target_pstate_use_performance(struct cpudata *cpu) return cpu->pstate.current_pstate - pid_calc(&cpu->pid, perf_scaled); } -static inline void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) +static int intel_pstate_prepare_request(struct cpudata *cpu, int pstate) { int max_perf, min_perf; - update_turbo_state(); - intel_pstate_get_min_max(cpu, &min_perf, &max_perf); pstate = clamp_t(int, pstate, min_perf, max_perf); trace_cpu_frequency(pstate * cpu->pstate.scaling, cpu->cpu); + return pstate; +} + +static void intel_pstate_update_pstate(struct cpudata *cpu, int pstate) +{ + pstate = intel_pstate_prepare_request(cpu, pstate); if (pstate == cpu->pstate.current_pstate) return; @@ -1396,6 +1762,8 @@ static inline void intel_pstate_adjust_busy_pstate(struct cpudata *cpu) target_pstate = cpu->policy == CPUFREQ_POLICY_PERFORMANCE ? cpu->pstate.turbo_pstate : pstate_funcs.get_target_pstate(cpu); + update_turbo_state(); + intel_pstate_update_pstate(cpu, target_pstate); sample = &cpu->sample; @@ -1416,7 +1784,7 @@ static void intel_pstate_update_util(struct update_util_data *data, u64 time, struct cpudata *cpu = container_of(data, struct cpudata, update_util); u64 delta_ns; - if (pid_params.boost_iowait) { + if (pstate_funcs.get_target_pstate == get_target_pstate_use_cpu_load) { if (flags & SCHED_CPUFREQ_IOWAIT) { cpu->iowait_boost = int_tofp(1); } else if (cpu->iowait_boost) { @@ -1462,6 +1830,7 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = { ICPU(INTEL_FAM6_SKYLAKE_DESKTOP, core_params), ICPU(INTEL_FAM6_BROADWELL_XEON_D, core_params), ICPU(INTEL_FAM6_XEON_PHI_KNL, knl_params), + ICPU(INTEL_FAM6_XEON_PHI_KNM, knl_params), ICPU(INTEL_FAM6_ATOM_GOLDMONT, bxt_params), {} }; @@ -1478,11 +1847,26 @@ static int intel_pstate_init_cpu(unsigned int cpunum) { struct cpudata *cpu; - if (!all_cpu_data[cpunum]) - all_cpu_data[cpunum] = kzalloc(sizeof(struct cpudata), - GFP_KERNEL); - if (!all_cpu_data[cpunum]) - return -ENOMEM; + cpu = all_cpu_data[cpunum]; + + if (!cpu) { + unsigned int size = sizeof(struct cpudata); + + if (per_cpu_limits) + size += sizeof(struct perf_limits); + + cpu = kzalloc(size, GFP_KERNEL); + if (!cpu) + return -ENOMEM; + + all_cpu_data[cpunum] = cpu; + if (per_cpu_limits) + cpu->perf_limits = (struct perf_limits *)(cpu + 1); + + cpu->epp_default = -EINVAL; + cpu->epp_powersave = -EINVAL; + cpu->epp_saved = -EINVAL; + } cpu = all_cpu_data[cpunum]; @@ -1541,18 +1925,57 @@ static void intel_pstate_set_performance_limits(struct perf_limits *limits) limits->no_turbo = 0; limits->turbo_disabled = 0; limits->max_perf_pct = 100; - limits->max_perf = int_tofp(1); + limits->max_perf = int_ext_tofp(1); limits->min_perf_pct = 100; - limits->min_perf = int_tofp(1); + limits->min_perf = int_ext_tofp(1); limits->max_policy_pct = 100; limits->max_sysfs_pct = 100; limits->min_policy_pct = 0; limits->min_sysfs_pct = 0; } +static void intel_pstate_update_perf_limits(struct cpufreq_policy *policy, + struct perf_limits *limits) +{ + + limits->max_policy_pct = DIV_ROUND_UP(policy->max * 100, + policy->cpuinfo.max_freq); + limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0, 100); + if (policy->max == policy->min) { + limits->min_policy_pct = limits->max_policy_pct; + } else { + limits->min_policy_pct = DIV_ROUND_UP(policy->min * 100, + policy->cpuinfo.max_freq); + limits->min_policy_pct = clamp_t(int, limits->min_policy_pct, + 0, 100); + } + + /* Normalize user input to [min_policy_pct, max_policy_pct] */ + limits->min_perf_pct = max(limits->min_policy_pct, + limits->min_sysfs_pct); + limits->min_perf_pct = min(limits->max_policy_pct, + limits->min_perf_pct); + limits->max_perf_pct = min(limits->max_policy_pct, + limits->max_sysfs_pct); + limits->max_perf_pct = max(limits->min_policy_pct, + limits->max_perf_pct); + + /* Make sure min_perf_pct <= max_perf_pct */ + limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct); + + limits->min_perf = div_ext_fp(limits->min_perf_pct, 100); + limits->max_perf = div_ext_fp(limits->max_perf_pct, 100); + limits->max_perf = round_up(limits->max_perf, EXT_FRAC_BITS); + limits->min_perf = round_up(limits->min_perf, EXT_FRAC_BITS); + + pr_debug("cpu:%d max_perf_pct:%d min_perf_pct:%d\n", policy->cpu, + limits->max_perf_pct, limits->min_perf_pct); +} + static int intel_pstate_set_policy(struct cpufreq_policy *policy) { struct cpudata *cpu; + struct perf_limits *perf_limits = NULL; if (!policy->cpuinfo.max_freq) return -ENODEV; @@ -1570,41 +1993,31 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) policy->max = policy->cpuinfo.max_freq; } - if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { - limits = &performance_limits; + if (per_cpu_limits) + perf_limits = cpu->perf_limits; + + mutex_lock(&intel_pstate_limits_lock); + + if (policy->policy == CPUFREQ_POLICY_PERFORMANCE) { + if (!perf_limits) { + limits = &performance_limits; + perf_limits = limits; + } if (policy->max >= policy->cpuinfo.max_freq) { pr_debug("set performance\n"); - intel_pstate_set_performance_limits(limits); + intel_pstate_set_performance_limits(perf_limits); goto out; } } else { pr_debug("set powersave\n"); - limits = &powersave_limits; - } - - limits->min_policy_pct = (policy->min * 100) / policy->cpuinfo.max_freq; - limits->min_policy_pct = clamp_t(int, limits->min_policy_pct, 0 , 100); - limits->max_policy_pct = DIV_ROUND_UP(policy->max * 100, - policy->cpuinfo.max_freq); - limits->max_policy_pct = clamp_t(int, limits->max_policy_pct, 0 , 100); - - /* Normalize user input to [min_policy_pct, max_policy_pct] */ - limits->min_perf_pct = max(limits->min_policy_pct, - limits->min_sysfs_pct); - limits->min_perf_pct = min(limits->max_policy_pct, - limits->min_perf_pct); - limits->max_perf_pct = min(limits->max_policy_pct, - limits->max_sysfs_pct); - limits->max_perf_pct = max(limits->min_policy_pct, - limits->max_perf_pct); - - /* Make sure min_perf_pct <= max_perf_pct */ - limits->min_perf_pct = min(limits->max_perf_pct, limits->min_perf_pct); + if (!perf_limits) { + limits = &powersave_limits; + perf_limits = limits; + } - limits->min_perf = div_fp(limits->min_perf_pct, 100); - limits->max_perf = div_fp(limits->max_perf_pct, 100); - limits->max_perf = round_up(limits->max_perf, FRAC_BITS); + } + intel_pstate_update_perf_limits(policy, perf_limits); out: if (cpu->policy == CPUFREQ_POLICY_PERFORMANCE) { /* @@ -1619,6 +2032,8 @@ static int intel_pstate_set_policy(struct cpufreq_policy *policy) intel_pstate_hwp_set_policy(policy); + mutex_unlock(&intel_pstate_limits_lock); + return 0; } @@ -1633,22 +2048,32 @@ static int intel_pstate_verify_policy(struct cpufreq_policy *policy) return 0; } +static void intel_cpufreq_stop_cpu(struct cpufreq_policy *policy) +{ + intel_pstate_set_min_pstate(all_cpu_data[policy->cpu]); +} + static void intel_pstate_stop_cpu(struct cpufreq_policy *policy) { - int cpu_num = policy->cpu; - struct cpudata *cpu = all_cpu_data[cpu_num]; + pr_debug("CPU %d exiting\n", policy->cpu); - pr_debug("CPU %d exiting\n", cpu_num); + intel_pstate_clear_update_util_hook(policy->cpu); + if (hwp_active) + intel_pstate_hwp_save_state(policy); + else + intel_cpufreq_stop_cpu(policy); +} - intel_pstate_clear_update_util_hook(cpu_num); +static int intel_pstate_cpu_exit(struct cpufreq_policy *policy) +{ + intel_pstate_exit_perf_limits(policy); - if (hwp_active) - return; + policy->fast_switch_possible = false; - intel_pstate_set_min_pstate(cpu); + return 0; } -static int intel_pstate_cpu_init(struct cpufreq_policy *policy) +static int __intel_pstate_cpu_init(struct cpufreq_policy *policy) { struct cpudata *cpu; int rc; @@ -1659,10 +2084,13 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy) cpu = all_cpu_data[policy->cpu]; - if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100) - policy->policy = CPUFREQ_POLICY_PERFORMANCE; - else - policy->policy = CPUFREQ_POLICY_POWERSAVE; + /* + * We need sane value in the cpu->perf_limits, so inherit from global + * perf_limits limits, which are seeded with values based on the + * CONFIG_CPU_FREQ_DEFAULT_GOV_*, during boot up. + */ + if (per_cpu_limits) + memcpy(cpu->perf_limits, limits, sizeof(struct perf_limits)); policy->min = cpu->pstate.min_pstate * cpu->pstate.scaling; policy->max = cpu->pstate.turbo_pstate * cpu->pstate.scaling; @@ -1675,24 +2103,35 @@ static int intel_pstate_cpu_init(struct cpufreq_policy *policy) policy->cpuinfo.max_freq *= cpu->pstate.scaling; intel_pstate_init_acpi_perf_limits(policy); - policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; cpumask_set_cpu(policy->cpu, policy->cpus); + policy->fast_switch_possible = true; + return 0; } -static int intel_pstate_cpu_exit(struct cpufreq_policy *policy) +static int intel_pstate_cpu_init(struct cpufreq_policy *policy) { - intel_pstate_exit_perf_limits(policy); + int ret = __intel_pstate_cpu_init(policy); + + if (ret) + return ret; + + policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; + if (limits->min_perf_pct == 100 && limits->max_perf_pct == 100) + policy->policy = CPUFREQ_POLICY_PERFORMANCE; + else + policy->policy = CPUFREQ_POLICY_POWERSAVE; return 0; } -static struct cpufreq_driver intel_pstate_driver = { +static struct cpufreq_driver intel_pstate = { .flags = CPUFREQ_CONST_LOOPS, .verify = intel_pstate_verify_policy, .setpolicy = intel_pstate_set_policy, - .resume = intel_pstate_hwp_set_policy, + .suspend = intel_pstate_hwp_save_state, + .resume = intel_pstate_resume, .get = intel_pstate_get, .init = intel_pstate_cpu_init, .exit = intel_pstate_cpu_exit, @@ -1700,6 +2139,118 @@ static struct cpufreq_driver intel_pstate_driver = { .name = "intel_pstate", }; +static int intel_cpufreq_verify_policy(struct cpufreq_policy *policy) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + struct perf_limits *perf_limits = limits; + + update_turbo_state(); + policy->cpuinfo.max_freq = limits->turbo_disabled ? + cpu->pstate.max_freq : cpu->pstate.turbo_freq; + + cpufreq_verify_within_cpu_limits(policy); + + if (per_cpu_limits) + perf_limits = cpu->perf_limits; + + intel_pstate_update_perf_limits(policy, perf_limits); + + return 0; +} + +static unsigned int intel_cpufreq_turbo_update(struct cpudata *cpu, + struct cpufreq_policy *policy, + unsigned int target_freq) +{ + unsigned int max_freq; + + update_turbo_state(); + + max_freq = limits->no_turbo || limits->turbo_disabled ? + cpu->pstate.max_freq : cpu->pstate.turbo_freq; + policy->cpuinfo.max_freq = max_freq; + if (policy->max > max_freq) + policy->max = max_freq; + + if (target_freq > max_freq) + target_freq = max_freq; + + return target_freq; +} + +static int intel_cpufreq_target(struct cpufreq_policy *policy, + unsigned int target_freq, + unsigned int relation) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + struct cpufreq_freqs freqs; + int target_pstate; + + freqs.old = policy->cur; + freqs.new = intel_cpufreq_turbo_update(cpu, policy, target_freq); + + cpufreq_freq_transition_begin(policy, &freqs); + switch (relation) { + case CPUFREQ_RELATION_L: + target_pstate = DIV_ROUND_UP(freqs.new, cpu->pstate.scaling); + break; + case CPUFREQ_RELATION_H: + target_pstate = freqs.new / cpu->pstate.scaling; + break; + default: + target_pstate = DIV_ROUND_CLOSEST(freqs.new, cpu->pstate.scaling); + break; + } + target_pstate = intel_pstate_prepare_request(cpu, target_pstate); + if (target_pstate != cpu->pstate.current_pstate) { + cpu->pstate.current_pstate = target_pstate; + wrmsrl_on_cpu(policy->cpu, MSR_IA32_PERF_CTL, + pstate_funcs.get_val(cpu, target_pstate)); + } + cpufreq_freq_transition_end(policy, &freqs, false); + + return 0; +} + +static unsigned int intel_cpufreq_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + struct cpudata *cpu = all_cpu_data[policy->cpu]; + int target_pstate; + + target_freq = intel_cpufreq_turbo_update(cpu, policy, target_freq); + target_pstate = DIV_ROUND_UP(target_freq, cpu->pstate.scaling); + intel_pstate_update_pstate(cpu, target_pstate); + return target_freq; +} + +static int intel_cpufreq_cpu_init(struct cpufreq_policy *policy) +{ + int ret = __intel_pstate_cpu_init(policy); + + if (ret) + return ret; + + policy->cpuinfo.transition_latency = INTEL_CPUFREQ_TRANSITION_LATENCY; + /* This reflects the intel_pstate_get_cpu_pstates() setting. */ + policy->cur = policy->cpuinfo.min_freq; + + return 0; +} + +static struct cpufreq_driver intel_cpufreq = { + .flags = CPUFREQ_CONST_LOOPS, + .verify = intel_cpufreq_verify_policy, + .target = intel_cpufreq_target, + .fast_switch = intel_cpufreq_fast_switch, + .init = intel_cpufreq_cpu_init, + .exit = intel_pstate_cpu_exit, + .stop_cpu = intel_cpufreq_stop_cpu, + .name = "intel_cpufreq", +}; + +static struct cpufreq_driver *intel_pstate_driver = &intel_pstate; + static int no_load __initdata; static int no_hwp __initdata; static int hwp_only __initdata; @@ -1726,6 +2277,19 @@ static void __init copy_pid_params(struct pstate_adjust_policy *policy) pid_params.setpoint = policy->setpoint; } +#ifdef CONFIG_ACPI +static void intel_pstate_use_acpi_profile(void) +{ + if (acpi_gbl_FADT.preferred_profile == PM_MOBILE) + pstate_funcs.get_target_pstate = + get_target_pstate_use_cpu_load; +} +#else +static void intel_pstate_use_acpi_profile(void) +{ +} +#endif + static void __init copy_cpu_funcs(struct pstate_funcs *funcs) { pstate_funcs.get_max = funcs->get_max; @@ -1737,6 +2301,7 @@ static void __init copy_cpu_funcs(struct pstate_funcs *funcs) pstate_funcs.get_vid = funcs->get_vid; pstate_funcs.get_target_pstate = funcs->get_target_pstate; + intel_pstate_use_acpi_profile(); } #ifdef CONFIG_ACPI @@ -1850,9 +2415,20 @@ static bool __init intel_pstate_platform_pwr_mgmt_exists(void) return false; } + +static void intel_pstate_request_control_from_smm(void) +{ + /* + * It may be unsafe to request P-states control from SMM if _PPC support + * has not been enabled. + */ + if (acpi_ppc) + acpi_processor_pstate_control(); +} #else /* CONFIG_ACPI not enabled */ static inline bool intel_pstate_platform_pwr_mgmt_exists(void) { return false; } static inline bool intel_pstate_has_acpi_ppc(void) { return false; } +static inline void intel_pstate_request_control_from_smm(void) {} #endif /* CONFIG_ACPI */ static const struct x86_cpu_id hwp_support_ids[] __initconst = { @@ -1872,6 +2448,7 @@ static int __init intel_pstate_init(void) if (x86_match_cpu(hwp_support_ids) && !no_hwp) { copy_cpu_funcs(&core_params.funcs); hwp_active++; + intel_pstate.attr = hwp_cpufreq_attrs; goto hwp_cpu_matched; } @@ -1904,7 +2481,9 @@ hwp_cpu_matched: if (!hwp_active && hwp_only) goto out; - rc = cpufreq_register_driver(&intel_pstate_driver); + intel_pstate_request_control_from_smm(); + + rc = cpufreq_register_driver(intel_pstate_driver); if (rc) goto out; @@ -1919,7 +2498,9 @@ out: get_online_cpus(); for_each_online_cpu(cpu) { if (all_cpu_data[cpu]) { - intel_pstate_clear_update_util_hook(cpu); + if (intel_pstate_driver == &intel_pstate) + intel_pstate_clear_update_util_hook(cpu); + kfree(all_cpu_data[cpu]); } } @@ -1935,8 +2516,13 @@ static int __init intel_pstate_setup(char *str) if (!str) return -EINVAL; - if (!strcmp(str, "disable")) + if (!strcmp(str, "disable")) { no_load = 1; + } else if (!strcmp(str, "passive")) { + pr_info("Passive mode enabled\n"); + intel_pstate_driver = &intel_cpufreq; + no_hwp = 1; + } if (!strcmp(str, "no_hwp")) { pr_info("HWP disabled\n"); no_hwp = 1; @@ -1945,6 +2531,8 @@ static int __init intel_pstate_setup(char *str) force_load = 1; if (!strcmp(str, "hwp_only")) hwp_only = 1; + if (!strcmp(str, "per_cpu_perf_limits")) + per_cpu_limits = true; #ifdef CONFIG_ACPI if (!strcmp(str, "support_acpi_ppc")) diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c index d3ffde806629..37671b545880 100644 --- a/drivers/cpufreq/powernv-cpufreq.c +++ b/drivers/cpufreq/powernv-cpufreq.c @@ -42,6 +42,10 @@ #define PMSR_PSAFE_ENABLE (1UL << 30) #define PMSR_SPR_EM_DISABLE (1UL << 31) #define PMSR_MAX(x) ((x >> 32) & 0xFF) +#define LPSTATE_SHIFT 48 +#define GPSTATE_SHIFT 56 +#define GET_LPSTATE(x) (((x) >> LPSTATE_SHIFT) & 0xFF) +#define GET_GPSTATE(x) (((x) >> GPSTATE_SHIFT) & 0xFF) #define MAX_RAMP_DOWN_TIME 5120 /* @@ -592,7 +596,8 @@ void gpstate_timer_handler(unsigned long data) { struct cpufreq_policy *policy = (struct cpufreq_policy *)data; struct global_pstate_info *gpstates = policy->driver_data; - int gpstate_idx; + int gpstate_idx, lpstate_idx; + unsigned long val; unsigned int time_diff = jiffies_to_msecs(jiffies) - gpstates->last_sampled_time; struct powernv_smp_call_data freq_data; @@ -600,21 +605,37 @@ void gpstate_timer_handler(unsigned long data) if (!spin_trylock(&gpstates->gpstate_lock)) return; + /* + * If PMCR was last updated was using fast_swtich then + * We may have wrong in gpstate->last_lpstate_idx + * value. Hence, read from PMCR to get correct data. + */ + val = get_pmspr(SPRN_PMCR); + freq_data.gpstate_id = (s8)GET_GPSTATE(val); + freq_data.pstate_id = (s8)GET_LPSTATE(val); + if (freq_data.gpstate_id == freq_data.pstate_id) { + reset_gpstates(policy); + spin_unlock(&gpstates->gpstate_lock); + return; + } + gpstates->last_sampled_time += time_diff; gpstates->elapsed_time += time_diff; - freq_data.pstate_id = idx_to_pstate(gpstates->last_lpstate_idx); - if ((gpstates->last_gpstate_idx == gpstates->last_lpstate_idx) || - (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME)) { + if (gpstates->elapsed_time > MAX_RAMP_DOWN_TIME) { gpstate_idx = pstate_to_idx(freq_data.pstate_id); + lpstate_idx = gpstate_idx; reset_gpstates(policy); gpstates->highest_lpstate_idx = gpstate_idx; } else { + lpstate_idx = pstate_to_idx(freq_data.pstate_id); gpstate_idx = calc_global_pstate(gpstates->elapsed_time, gpstates->highest_lpstate_idx, - gpstates->last_lpstate_idx); + lpstate_idx); } - + freq_data.gpstate_id = idx_to_pstate(gpstate_idx); + gpstates->last_gpstate_idx = gpstate_idx; + gpstates->last_lpstate_idx = lpstate_idx; /* * If local pstate is equal to global pstate, rampdown is over * So timer is not required to be queued. @@ -622,10 +643,6 @@ void gpstate_timer_handler(unsigned long data) if (gpstate_idx != gpstates->last_lpstate_idx) queue_gpstate_timer(gpstates); - freq_data.gpstate_id = idx_to_pstate(gpstate_idx); - gpstates->last_gpstate_idx = pstate_to_idx(freq_data.gpstate_id); - gpstates->last_lpstate_idx = pstate_to_idx(freq_data.pstate_id); - spin_unlock(&gpstates->gpstate_lock); /* Timer may get migrated to a different cpu on cpu hot unplug */ @@ -647,8 +664,14 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, if (unlikely(rebooting) && new_index != get_nominal_index()) return 0; - if (!throttled) + if (!throttled) { + /* we don't want to be preempted while + * checking if the CPU frequency has been throttled + */ + preempt_disable(); powernv_cpufreq_throttle_check(NULL); + preempt_enable(); + } cur_msec = jiffies_to_msecs(get_jiffies_64()); @@ -752,9 +775,12 @@ static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy) spin_lock_init(&gpstates->gpstate_lock); ret = cpufreq_table_validate_and_show(policy, powernv_freqs); - if (ret < 0) + if (ret < 0) { kfree(policy->driver_data); + return ret; + } + policy->fast_switch_possible = true; return ret; } @@ -897,6 +923,20 @@ static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy) del_timer_sync(&gpstates->timer); } +static unsigned int powernv_fast_switch(struct cpufreq_policy *policy, + unsigned int target_freq) +{ + int index; + struct powernv_smp_call_data freq_data; + + index = cpufreq_table_find_index_dl(policy, target_freq); + freq_data.pstate_id = powernv_freqs[index].driver_data; + freq_data.gpstate_id = powernv_freqs[index].driver_data; + set_pstate(&freq_data); + + return powernv_freqs[index].frequency; +} + static struct cpufreq_driver powernv_cpufreq_driver = { .name = "powernv-cpufreq", .flags = CPUFREQ_CONST_LOOPS, @@ -904,6 +944,7 @@ static struct cpufreq_driver powernv_cpufreq_driver = { .exit = powernv_cpufreq_cpu_exit, .verify = cpufreq_generic_frequency_table_verify, .target_index = powernv_cpufreq_target_index, + .fast_switch = powernv_fast_switch, .get = powernv_cpufreq_get, .stop_cpu = powernv_cpufreq_stop_cpu, .attr = powernv_cpu_freq_attr, diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c index 7fe442ca38f4..0835a37a5f3a 100644 --- a/drivers/cpuidle/cpuidle-powernv.c +++ b/drivers/cpuidle/cpuidle-powernv.c @@ -22,7 +22,7 @@ #define POWERNV_THRESHOLD_LATENCY_NS 200000 -struct cpuidle_driver powernv_idle_driver = { +static struct cpuidle_driver powernv_idle_driver = { .name = "powernv_idle", .owner = THIS_MODULE, }; diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index c73207abb5a4..62810ff3b00f 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -97,7 +97,23 @@ static int find_deepest_state(struct cpuidle_driver *drv, return ret; } -#ifdef CONFIG_SUSPEND +/** + * cpuidle_use_deepest_state - Set/clear governor override flag. + * @enable: New value of the flag. + * + * Set/unset the current CPU to use the deepest idle state (override governors + * going forward if set). + */ +void cpuidle_use_deepest_state(bool enable) +{ + struct cpuidle_device *dev; + + preempt_disable(); + dev = cpuidle_get_device(); + dev->use_deepest_state = enable; + preempt_enable(); +} + /** * cpuidle_find_deepest_state - Find the deepest available idle state. * @drv: cpuidle driver for the given CPU. @@ -109,6 +125,7 @@ int cpuidle_find_deepest_state(struct cpuidle_driver *drv, return find_deepest_state(drv, dev, UINT_MAX, 0, false); } +#ifdef CONFIG_SUSPEND static void enter_freeze_proper(struct cpuidle_driver *drv, struct cpuidle_device *dev, int index) { diff --git a/drivers/cpuidle/dt_idle_states.c b/drivers/cpuidle/dt_idle_states.c index a5c111b67f37..ffca4fc0061d 100644 --- a/drivers/cpuidle/dt_idle_states.c +++ b/drivers/cpuidle/dt_idle_states.c @@ -38,6 +38,12 @@ static int init_state_node(struct cpuidle_state *idle_state, * state enter function. */ idle_state->enter = match_id->data; + /* + * Since this is not a "coupled" state, it's safe to assume interrupts + * won't be enabled when it exits allowing the tick to be frozen + * safely. So enter() can be also enter_freeze() callback. + */ + idle_state->enter_freeze = match_id->data; err = of_property_read_u32(state_node, "wakeup-latency-us", &idle_state->exit_latency); diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c index fb9f511cca23..4e78263e34a4 100644 --- a/drivers/cpuidle/governor.c +++ b/drivers/cpuidle/governor.c @@ -9,7 +9,6 @@ */ #include <linux/mutex.h> -#include <linux/module.h> #include <linux/cpuidle.h> #include "cpuidle.h" @@ -53,14 +52,11 @@ int cpuidle_switch_governor(struct cpuidle_governor *gov) if (cpuidle_curr_governor) { list_for_each_entry(dev, &cpuidle_detected_devices, device_list) cpuidle_disable_device(dev); - module_put(cpuidle_curr_governor->owner); } cpuidle_curr_governor = gov; if (gov) { - if (!try_module_get(cpuidle_curr_governor->owner)) - return -EINVAL; list_for_each_entry(dev, &cpuidle_detected_devices, device_list) cpuidle_enable_device(dev); cpuidle_install_idle_handler(); diff --git a/drivers/cpuidle/governors/ladder.c b/drivers/cpuidle/governors/ladder.c index 63bd5a403e22..fe8f08948fcb 100644 --- a/drivers/cpuidle/governors/ladder.c +++ b/drivers/cpuidle/governors/ladder.c @@ -15,7 +15,6 @@ #include <linux/kernel.h> #include <linux/cpuidle.h> #include <linux/pm_qos.h> -#include <linux/module.h> #include <linux/jiffies.h> #include <linux/tick.h> @@ -177,7 +176,6 @@ static struct cpuidle_governor ladder_governor = { .enable = ladder_enable_device, .select = ladder_select_state, .reflect = ladder_reflect, - .owner = THIS_MODULE, }; /** diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 03d38c291de6..d9b5b9398a0f 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -19,7 +19,6 @@ #include <linux/tick.h> #include <linux/sched.h> #include <linux/math64.h> -#include <linux/module.h> /* * Please note when changing the tuning values: @@ -484,7 +483,6 @@ static struct cpuidle_governor menu_governor = { .enable = menu_enable_device, .select = menu_select, .reflect = menu_reflect, - .owner = THIS_MODULE, }; /** diff --git a/drivers/cpuidle/sysfs.c b/drivers/cpuidle/sysfs.c index 832a2c3f01ff..c5adc8c9ac43 100644 --- a/drivers/cpuidle/sysfs.c +++ b/drivers/cpuidle/sysfs.c @@ -403,8 +403,10 @@ static int cpuidle_add_state_sysfs(struct cpuidle_device *device) /* state statistics */ for (i = 0; i < drv->state_count; i++) { kobj = kzalloc(sizeof(struct cpuidle_state_kobj), GFP_KERNEL); - if (!kobj) + if (!kobj) { + ret = -ENOMEM; goto error_state; + } kobj->state = &drv->states[i]; kobj->state_usage = &device->states_usage[i]; init_completion(&kobj->kobj_unregister); diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index bf3ea7603a58..a324801d6a66 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -850,7 +850,7 @@ err_out: EXPORT_SYMBOL(devfreq_add_governor); /** - * devfreq_remove_device() - Remove devfreq feature from a device. + * devfreq_remove_governor() - Remove devfreq feature from a device. * @governor: the devfreq governor to be removed */ int devfreq_remove_governor(struct devfreq_governor *governor) diff --git a/drivers/devfreq/event/exynos-nocp.c b/drivers/devfreq/event/exynos-nocp.c index 49e712aca0c1..5c3e7b11e8a6 100644 --- a/drivers/devfreq/event/exynos-nocp.c +++ b/drivers/devfreq/event/exynos-nocp.c @@ -190,6 +190,7 @@ static const struct of_device_id exynos_nocp_id_match[] = { { .compatible = "samsung,exynos5420-nocp", }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, exynos_nocp_id_match); static struct regmap_config exynos_nocp_regmap_config = { .reg_bits = 32, diff --git a/drivers/devfreq/event/exynos-ppmu.c b/drivers/devfreq/event/exynos-ppmu.c index f55cf0eb2a66..107eb91a9415 100644 --- a/drivers/devfreq/event/exynos-ppmu.c +++ b/drivers/devfreq/event/exynos-ppmu.c @@ -15,7 +15,6 @@ #include <linux/io.h> #include <linux/kernel.h> #include <linux/module.h> -#include <linux/mutex.h> #include <linux/of_address.h> #include <linux/platform_device.h> #include <linux/suspend.h> @@ -34,7 +33,6 @@ struct exynos_ppmu { unsigned int num_events; struct device *dev; - struct mutex lock; struct exynos_ppmu_data ppmu; }; @@ -90,8 +88,6 @@ struct __exynos_ppmu_events { PPMU_EVENT(d1-cpu), PPMU_EVENT(d1-general), PPMU_EVENT(d1-rt), - - { /* sentinel */ }, }; static int exynos_ppmu_find_ppmu_id(struct devfreq_event_dev *edev) @@ -351,6 +347,7 @@ static const struct of_device_id exynos_ppmu_id_match[] = { }, { /* sentinel */ }, }; +MODULE_DEVICE_TABLE(of, exynos_ppmu_id_match); static struct devfreq_event_ops *exynos_bus_get_ops(struct device_node *np) { @@ -463,7 +460,6 @@ static int exynos_ppmu_probe(struct platform_device *pdev) if (!info) return -ENOMEM; - mutex_init(&info->lock); info->dev = &pdev->dev; /* Parse dt data to get resource */ diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c index 43fcc5a7f515..22b113363ffc 100644 --- a/drivers/devfreq/event/rockchip-dfi.c +++ b/drivers/devfreq/event/rockchip-dfi.c @@ -188,6 +188,7 @@ static const struct of_device_id rockchip_dfi_id_match[] = { { .compatible = "rockchip,rk3399-dfi" }, { }, }; +MODULE_DEVICE_TABLE(of, rockchip_dfi_id_match); static int rockchip_dfi_probe(struct platform_device *pdev) { diff --git a/drivers/devfreq/exynos-bus.c b/drivers/devfreq/exynos-bus.c index 29866f7e6d7e..a8ed7792ece2 100644 --- a/drivers/devfreq/exynos-bus.c +++ b/drivers/devfreq/exynos-bus.c @@ -35,7 +35,7 @@ struct exynos_bus { unsigned int edev_count; struct mutex lock; - struct dev_pm_opp *curr_opp; + unsigned long curr_freq; struct regulator *regulator; struct clk *clk; @@ -99,7 +99,7 @@ static int exynos_bus_target(struct device *dev, unsigned long *freq, u32 flags) { struct exynos_bus *bus = dev_get_drvdata(dev); struct dev_pm_opp *new_opp; - unsigned long old_freq, new_freq, old_volt, new_volt, tol; + unsigned long old_freq, new_freq, new_volt, tol; int ret = 0; /* Get new opp-bus instance according to new bus clock */ @@ -113,8 +113,7 @@ static int exynos_bus_target(struct device *dev, unsigned long *freq, u32 flags) new_freq = dev_pm_opp_get_freq(new_opp); new_volt = dev_pm_opp_get_voltage(new_opp); - old_freq = dev_pm_opp_get_freq(bus->curr_opp); - old_volt = dev_pm_opp_get_voltage(bus->curr_opp); + old_freq = bus->curr_freq; rcu_read_unlock(); if (old_freq == new_freq) @@ -146,7 +145,7 @@ static int exynos_bus_target(struct device *dev, unsigned long *freq, u32 flags) goto out; } } - bus->curr_opp = new_opp; + bus->curr_freq = new_freq; dev_dbg(dev, "Set the frequency of bus (%lukHz -> %lukHz)\n", old_freq/1000, new_freq/1000); @@ -163,9 +162,7 @@ static int exynos_bus_get_dev_status(struct device *dev, struct devfreq_event_data edata; int ret; - rcu_read_lock(); - stat->current_frequency = dev_pm_opp_get_freq(bus->curr_opp); - rcu_read_unlock(); + stat->current_frequency = bus->curr_freq; ret = exynos_bus_get_event(bus, &edata); if (ret < 0) { @@ -226,7 +223,7 @@ static int exynos_bus_passive_target(struct device *dev, unsigned long *freq, } new_freq = dev_pm_opp_get_freq(new_opp); - old_freq = dev_pm_opp_get_freq(bus->curr_opp); + old_freq = bus->curr_freq; rcu_read_unlock(); if (old_freq == new_freq) @@ -242,7 +239,7 @@ static int exynos_bus_passive_target(struct device *dev, unsigned long *freq, } *freq = new_freq; - bus->curr_opp = new_opp; + bus->curr_freq = new_freq; dev_dbg(dev, "Set the frequency of bus (%lukHz -> %lukHz)\n", old_freq/1000, new_freq/1000); @@ -335,6 +332,7 @@ static int exynos_bus_parse_of(struct device_node *np, struct exynos_bus *bus) { struct device *dev = bus->dev; + struct dev_pm_opp *opp; unsigned long rate; int ret; @@ -352,22 +350,23 @@ static int exynos_bus_parse_of(struct device_node *np, } /* Get the freq and voltage from OPP table to scale the bus freq */ - rcu_read_lock(); ret = dev_pm_opp_of_add_table(dev); if (ret < 0) { dev_err(dev, "failed to get OPP table\n"); - rcu_read_unlock(); goto err_clk; } rate = clk_get_rate(bus->clk); - bus->curr_opp = devfreq_recommended_opp(dev, &rate, 0); - if (IS_ERR(bus->curr_opp)) { + + rcu_read_lock(); + opp = devfreq_recommended_opp(dev, &rate, 0); + if (IS_ERR(opp)) { dev_err(dev, "failed to find dev_pm_opp\n"); rcu_read_unlock(); - ret = PTR_ERR(bus->curr_opp); + ret = PTR_ERR(opp); goto err_opp; } + bus->curr_freq = dev_pm_opp_get_freq(opp); rcu_read_unlock(); return 0; diff --git a/drivers/devfreq/rk3399_dmc.c b/drivers/devfreq/rk3399_dmc.c index e24b73d66659..27d2f349b53c 100644 --- a/drivers/devfreq/rk3399_dmc.c +++ b/drivers/devfreq/rk3399_dmc.c @@ -80,7 +80,6 @@ struct rk3399_dmcfreq { struct regulator *vdd_center; unsigned long rate, target_rate; unsigned long volt, target_volt; - struct dev_pm_opp *curr_opp; }; static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq, @@ -102,9 +101,6 @@ static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq, target_rate = dev_pm_opp_get_freq(opp); target_volt = dev_pm_opp_get_voltage(opp); - dmcfreq->rate = dev_pm_opp_get_freq(dmcfreq->curr_opp); - dmcfreq->volt = dev_pm_opp_get_voltage(dmcfreq->curr_opp); - rcu_read_unlock(); if (dmcfreq->rate == target_rate) @@ -165,7 +161,9 @@ static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq, if (err) dev_err(dev, "Cannot to set vol %lu uV\n", target_volt); - dmcfreq->curr_opp = opp; + dmcfreq->rate = target_rate; + dmcfreq->volt = target_volt; + out: mutex_unlock(&dmcfreq->lock); return err; @@ -414,7 +412,6 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev) */ if (dev_pm_opp_of_add_table(dev)) { dev_err(dev, "Invalid operating-points in device tree.\n"); - rcu_read_unlock(); return -EINVAL; } @@ -431,12 +428,13 @@ static int rk3399_dmcfreq_probe(struct platform_device *pdev) rcu_read_unlock(); return PTR_ERR(opp); } + data->rate = dev_pm_opp_get_freq(opp); + data->volt = dev_pm_opp_get_voltage(opp); rcu_read_unlock(); - data->curr_opp = opp; rk3399_devfreq_dmc_profile.initial_freq = data->rate; - data->devfreq = devfreq_add_device(dev, + data->devfreq = devm_devfreq_add_device(dev, &rk3399_devfreq_dmc_profile, "simple_ondemand", &data->ondemand_data); @@ -454,6 +452,7 @@ static const struct of_device_id rk3399dmc_devfreq_of_match[] = { { .compatible = "rockchip,rk3399-dmc" }, { }, }; +MODULE_DEVICE_TABLE(of, rk3399dmc_devfreq_of_match); static struct platform_driver rk3399_dmcfreq_driver = { .probe = rk3399_dmcfreq_probe, diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c index 4466a2f969d7..7d8ea3d5fda6 100644 --- a/drivers/idle/intel_idle.c +++ b/drivers/idle/intel_idle.c @@ -98,8 +98,6 @@ static int intel_idle(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index); static void intel_idle_freeze(struct cpuidle_device *dev, struct cpuidle_driver *drv, int index); -static int intel_idle_cpu_init(int cpu); - static struct cpuidle_state *cpuidle_state_table; /* @@ -724,6 +722,50 @@ static struct cpuidle_state atom_cstates[] = { { .enter = NULL } }; +static struct cpuidle_state tangier_cstates[] = { + { + .name = "C1-TNG", + .desc = "MWAIT 0x00", + .flags = MWAIT2flg(0x00), + .exit_latency = 1, + .target_residency = 4, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C4-TNG", + .desc = "MWAIT 0x30", + .flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 100, + .target_residency = 400, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C6-TNG", + .desc = "MWAIT 0x52", + .flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 140, + .target_residency = 560, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C7-TNG", + .desc = "MWAIT 0x60", + .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 1200, + .target_residency = 4000, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .name = "C9-TNG", + .desc = "MWAIT 0x64", + .flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED, + .exit_latency = 10000, + .target_residency = 20000, + .enter = &intel_idle, + .enter_freeze = intel_idle_freeze, }, + { + .enter = NULL } +}; static struct cpuidle_state avn_cstates[] = { { .name = "C1-AVN", @@ -907,51 +949,15 @@ static void intel_idle_freeze(struct cpuidle_device *dev, mwait_idle_with_hints(eax, ecx); } -static void __setup_broadcast_timer(void *arg) +static void __setup_broadcast_timer(bool on) { - unsigned long on = (unsigned long)arg; - if (on) tick_broadcast_enable(); else tick_broadcast_disable(); } -static int cpu_hotplug_notify(struct notifier_block *n, - unsigned long action, void *hcpu) -{ - int hotcpu = (unsigned long)hcpu; - struct cpuidle_device *dev; - - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_ONLINE: - - if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) - smp_call_function_single(hotcpu, __setup_broadcast_timer, - (void *)true, 1); - - /* - * Some systems can hotplug a cpu at runtime after - * the kernel has booted, we have to initialize the - * driver in this case - */ - dev = per_cpu_ptr(intel_idle_cpuidle_devices, hotcpu); - if (dev->registered) - break; - - if (intel_idle_cpu_init(hotcpu)) - return NOTIFY_BAD; - - break; - } - return NOTIFY_OK; -} - -static struct notifier_block cpu_hotplug_notifier = { - .notifier_call = cpu_hotplug_notify, -}; - -static void auto_demotion_disable(void *dummy) +static void auto_demotion_disable(void) { unsigned long long msr_bits; @@ -959,7 +965,7 @@ static void auto_demotion_disable(void *dummy) msr_bits &= ~(icpu->auto_demotion_disable_flags); wrmsrl(MSR_NHM_SNB_PKG_CST_CFG_CTL, msr_bits); } -static void c1e_promotion_disable(void *dummy) +static void c1e_promotion_disable(void) { unsigned long long msr_bits; @@ -978,6 +984,10 @@ static const struct idle_cpu idle_cpu_atom = { .state_table = atom_cstates, }; +static const struct idle_cpu idle_cpu_tangier = { + .state_table = tangier_cstates, +}; + static const struct idle_cpu idle_cpu_lincroft = { .state_table = atom_cstates, .auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE, @@ -1066,6 +1076,7 @@ static const struct x86_cpu_id intel_idle_ids[] __initconst = { ICPU(INTEL_FAM6_SANDYBRIDGE_X, idle_cpu_snb), ICPU(INTEL_FAM6_ATOM_CEDARVIEW, idle_cpu_atom), ICPU(INTEL_FAM6_ATOM_SILVERMONT1, idle_cpu_byt), + ICPU(INTEL_FAM6_ATOM_MERRIFIELD, idle_cpu_tangier), ICPU(INTEL_FAM6_ATOM_AIRMONT, idle_cpu_cht), ICPU(INTEL_FAM6_IVYBRIDGE, idle_cpu_ivb), ICPU(INTEL_FAM6_IVYBRIDGE_X, idle_cpu_ivt), @@ -1084,6 +1095,7 @@ static const struct x86_cpu_id intel_idle_ids[] __initconst = { ICPU(INTEL_FAM6_KABYLAKE_DESKTOP, idle_cpu_skl), ICPU(INTEL_FAM6_SKYLAKE_X, idle_cpu_skx), ICPU(INTEL_FAM6_XEON_PHI_KNL, idle_cpu_knl), + ICPU(INTEL_FAM6_XEON_PHI_KNM, idle_cpu_knl), ICPU(INTEL_FAM6_ATOM_GOLDMONT, idle_cpu_bxt), ICPU(INTEL_FAM6_ATOM_DENVERTON, idle_cpu_dnv), {} @@ -1373,12 +1385,11 @@ static void __init intel_idle_cpuidle_driver_init(void) * allocate, initialize, register cpuidle_devices * @cpu: cpu/core to initialize */ -static int intel_idle_cpu_init(int cpu) +static int intel_idle_cpu_init(unsigned int cpu) { struct cpuidle_device *dev; dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); - dev->cpu = cpu; if (cpuidle_register_device(dev)) { @@ -1387,17 +1398,36 @@ static int intel_idle_cpu_init(int cpu) } if (icpu->auto_demotion_disable_flags) - smp_call_function_single(cpu, auto_demotion_disable, NULL, 1); + auto_demotion_disable(); if (icpu->disable_promotion_to_c1e) - smp_call_function_single(cpu, c1e_promotion_disable, NULL, 1); + c1e_promotion_disable(); + + return 0; +} + +static int intel_idle_cpu_online(unsigned int cpu) +{ + struct cpuidle_device *dev; + + if (lapic_timer_reliable_states != LAPIC_TIMER_ALWAYS_RELIABLE) + __setup_broadcast_timer(true); + + /* + * Some systems can hotplug a cpu at runtime after + * the kernel has booted, we have to initialize the + * driver in this case + */ + dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu); + if (!dev->registered) + return intel_idle_cpu_init(cpu); return 0; } static int __init intel_idle_init(void) { - int retval, i; + int retval; /* Do not load intel_idle at all for now if idle= is passed */ if (boot_option_idle_override != IDLE_NO_OVERRIDE) @@ -1417,35 +1447,29 @@ static int __init intel_idle_init(void) struct cpuidle_driver *drv = cpuidle_get_driver(); printk(KERN_DEBUG PREFIX "intel_idle yielding to %s", drv ? drv->name : "none"); - free_percpu(intel_idle_cpuidle_devices); - return retval; + goto init_driver_fail; } - cpu_notifier_register_begin(); - - for_each_online_cpu(i) { - retval = intel_idle_cpu_init(i); - if (retval) { - intel_idle_cpuidle_devices_uninit(); - cpu_notifier_register_done(); - cpuidle_unregister_driver(&intel_idle_driver); - free_percpu(intel_idle_cpuidle_devices); - return retval; - } - } - __register_cpu_notifier(&cpu_hotplug_notifier); - if (boot_cpu_has(X86_FEATURE_ARAT)) /* Always Reliable APIC Timer */ lapic_timer_reliable_states = LAPIC_TIMER_ALWAYS_RELIABLE; - else - on_each_cpu(__setup_broadcast_timer, (void *)true, 1); - cpu_notifier_register_done(); + retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online", + intel_idle_cpu_online, NULL); + if (retval < 0) + goto hp_setup_fail; pr_debug(PREFIX "lapic_timer_reliable_states 0x%x\n", lapic_timer_reliable_states); return 0; + +hp_setup_fail: + intel_idle_cpuidle_devices_uninit(); + cpuidle_unregister_driver(&intel_idle_driver); +init_driver_fail: + free_percpu(intel_idle_cpuidle_devices); + return retval; + } device_initcall(intel_idle_init); diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c index c48fc0c4abd9..fa5ca0992be6 100644 --- a/drivers/net/ethernet/smsc/smsc911x.c +++ b/drivers/net/ethernet/smsc/smsc911x.c @@ -2585,6 +2585,9 @@ static int smsc911x_suspend(struct device *dev) PMT_CTRL_PM_MODE_D1_ | PMT_CTRL_WOL_EN_ | PMT_CTRL_ED_EN_ | PMT_CTRL_PME_EN_); + pm_runtime_disable(dev); + pm_runtime_set_suspended(dev); + return 0; } @@ -2594,6 +2597,9 @@ static int smsc911x_resume(struct device *dev) struct smsc911x_data *pdata = netdev_priv(ndev); unsigned int to = 100; + pm_runtime_enable(dev); + pm_runtime_resume(dev); + /* Note 3.11 from the datasheet: * "When the LAN9220 is in a power saving state, a write of any * data to the BYTE_TEST register will wake-up the device." diff --git a/drivers/power/avs/rockchip-io-domain.c b/drivers/power/avs/rockchip-io-domain.c index 01b6d3f9b8fb..56bce1908be2 100644 --- a/drivers/power/avs/rockchip-io-domain.c +++ b/drivers/power/avs/rockchip-io-domain.c @@ -143,7 +143,7 @@ static int rockchip_iodomain_notify(struct notifier_block *nb, if (ret && event == REGULATOR_EVENT_PRE_VOLTAGE_CHANGE) return NOTIFY_BAD; - dev_info(supply->iod->dev, "Setting to %d done\n", uV); + dev_dbg(supply->iod->dev, "Setting to %d done\n", uV); return NOTIFY_OK; } diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c index 243b233ff31b..9a25110c4a46 100644 --- a/drivers/powercap/intel_rapl.c +++ b/drivers/powercap/intel_rapl.c @@ -189,14 +189,13 @@ struct rapl_package { unsigned int time_unit; struct rapl_domain *domains; /* array of domains, sized at runtime */ struct powercap_zone *power_zone; /* keep track of parent zone */ - int nr_cpus; /* active cpus on the package, topology info is lost during - * cpu hotplug. so we have to track ourselves. - */ unsigned long power_limit_irq; /* keep track of package power limit * notify interrupt enable status. */ struct list_head plist; int lead_cpu; /* one active cpu per package for access */ + /* Track active cpus */ + struct cpumask cpumask; }; struct rapl_defaults { @@ -275,18 +274,6 @@ static struct rapl_package *find_package_by_id(int id) return NULL; } -/* caller must hold cpu hotplug lock */ -static void rapl_cleanup_data(void) -{ - struct rapl_package *p, *tmp; - - list_for_each_entry_safe(p, tmp, &rapl_packages, plist) { - kfree(p->domains); - list_del(&p->plist); - kfree(p); - } -} - static int get_energy_counter(struct powercap_zone *power_zone, u64 *energy_raw) { struct rapl_domain *rd; @@ -442,6 +429,7 @@ static int contraint_to_pl(struct rapl_domain *rd, int cid) return i; } } + pr_err("Cannot find matching power limit for constraint %d\n", cid); return -EINVAL; } @@ -457,6 +445,10 @@ static int set_power_limit(struct powercap_zone *power_zone, int cid, get_online_cpus(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); + if (id < 0) { + ret = id; + goto set_exit; + } rp = rd->rp; @@ -496,6 +488,11 @@ static int get_current_power_limit(struct powercap_zone *power_zone, int cid, get_online_cpus(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); + if (id < 0) { + ret = id; + goto get_exit; + } + switch (rd->rpl[id].prim_id) { case PL1_ENABLE: prim = POWER_LIMIT1; @@ -512,6 +509,7 @@ static int get_current_power_limit(struct powercap_zone *power_zone, int cid, else *data = val; +get_exit: put_online_cpus(); return ret; @@ -527,6 +525,10 @@ static int set_time_window(struct powercap_zone *power_zone, int cid, get_online_cpus(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); + if (id < 0) { + ret = id; + goto set_time_exit; + } switch (rd->rpl[id].prim_id) { case PL1_ENABLE: @@ -538,6 +540,8 @@ static int set_time_window(struct powercap_zone *power_zone, int cid, default: ret = -EINVAL; } + +set_time_exit: put_online_cpus(); return ret; } @@ -552,6 +556,10 @@ static int get_time_window(struct powercap_zone *power_zone, int cid, u64 *data) get_online_cpus(); rd = power_zone_to_rapl_domain(power_zone); id = contraint_to_pl(rd, cid); + if (id < 0) { + ret = id; + goto get_time_exit; + } switch (rd->rpl[id].prim_id) { case PL1_ENABLE: @@ -566,6 +574,8 @@ static int get_time_window(struct powercap_zone *power_zone, int cid, u64 *data) } if (!ret) *data = val; + +get_time_exit: put_online_cpus(); return ret; @@ -707,7 +717,7 @@ static u64 rapl_unit_xlate(struct rapl_domain *rd, enum unit_type type, case ENERGY_UNIT: scale = ENERGY_UNIT_SCALE; /* per domain unit takes precedence */ - if (rd && rd->domain_energy_unit) + if (rd->domain_energy_unit) units = rd->domain_energy_unit; else units = rp->energy_unit; @@ -976,10 +986,20 @@ static void package_power_limit_irq_save(struct rapl_package *rp) smp_call_function_single(rp->lead_cpu, power_limit_irq_save_cpu, rp, 1); } -static void power_limit_irq_restore_cpu(void *info) +/* + * Restore per package power limit interrupt enable state. Called from cpu + * hotplug code on package removal. + */ +static void package_power_limit_irq_restore(struct rapl_package *rp) { - u32 l, h = 0; - struct rapl_package *rp = (struct rapl_package *)info; + u32 l, h; + + if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) + return; + + /* irq enable state not saved, nothing to restore */ + if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) + return; rdmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h); @@ -991,19 +1011,6 @@ static void power_limit_irq_restore_cpu(void *info) wrmsr_safe(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h); } -/* restore per package power limit interrupt enable state */ -static void package_power_limit_irq_restore(struct rapl_package *rp) -{ - if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN)) - return; - - /* irq enable state not saved, nothing to restore */ - if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) - return; - - smp_call_function_single(rp->lead_cpu, power_limit_irq_restore_cpu, rp, 1); -} - static void set_floor_freq_default(struct rapl_domain *rd, bool mode) { int nr_powerlimit = find_nr_power_limit(rd); @@ -1160,84 +1167,49 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { RAPL_CPU(INTEL_FAM6_ATOM_DENVERTON, rapl_defaults_core), RAPL_CPU(INTEL_FAM6_XEON_PHI_KNL, rapl_defaults_hsw_server), + RAPL_CPU(INTEL_FAM6_XEON_PHI_KNM, rapl_defaults_hsw_server), {} }; MODULE_DEVICE_TABLE(x86cpu, rapl_ids); -/* read once for all raw primitive data for all packages, domains */ -static void rapl_update_domain_data(void) +/* Read once for all raw primitive data for domains */ +static void rapl_update_domain_data(struct rapl_package *rp) { int dmn, prim; u64 val; - struct rapl_package *rp; - list_for_each_entry(rp, &rapl_packages, plist) { - for (dmn = 0; dmn < rp->nr_domains; dmn++) { - pr_debug("update package %d domain %s data\n", rp->id, - rp->domains[dmn].name); - /* exclude non-raw primitives */ - for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) - if (!rapl_read_data_raw(&rp->domains[dmn], prim, - rpi[prim].unit, - &val)) - rp->domains[dmn].rdd.primitives[prim] = - val; + for (dmn = 0; dmn < rp->nr_domains; dmn++) { + pr_debug("update package %d domain %s data\n", rp->id, + rp->domains[dmn].name); + /* exclude non-raw primitives */ + for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) { + if (!rapl_read_data_raw(&rp->domains[dmn], prim, + rpi[prim].unit, &val)) + rp->domains[dmn].rdd.primitives[prim] = val; } } } -static int rapl_unregister_powercap(void) +static void rapl_unregister_powercap(void) { - struct rapl_package *rp; - struct rapl_domain *rd, *rd_package = NULL; - - /* unregister all active rapl packages from the powercap layer, - * hotplug lock held - */ - list_for_each_entry(rp, &rapl_packages, plist) { - package_power_limit_irq_restore(rp); - - for (rd = rp->domains; rd < rp->domains + rp->nr_domains; - rd++) { - pr_debug("remove package, undo power limit on %d: %s\n", - rp->id, rd->name); - rapl_write_data_raw(rd, PL1_ENABLE, 0); - rapl_write_data_raw(rd, PL1_CLAMP, 0); - if (find_nr_power_limit(rd) > 1) { - rapl_write_data_raw(rd, PL2_ENABLE, 0); - rapl_write_data_raw(rd, PL2_CLAMP, 0); - } - if (rd->id == RAPL_DOMAIN_PACKAGE) { - rd_package = rd; - continue; - } - powercap_unregister_zone(control_type, &rd->power_zone); - } - /* do the package zone last */ - if (rd_package) - powercap_unregister_zone(control_type, - &rd_package->power_zone); - } - if (platform_rapl_domain) { powercap_unregister_zone(control_type, &platform_rapl_domain->power_zone); kfree(platform_rapl_domain); } - powercap_unregister_control_type(control_type); - - return 0; } static int rapl_package_register_powercap(struct rapl_package *rp) { struct rapl_domain *rd; - int ret = 0; char dev_name[17]; /* max domain name = 7 + 1 + 8 for int + 1 for null*/ struct powercap_zone *power_zone = NULL; - int nr_pl; + int nr_pl, ret;; + + /* Update the domain data of the new package */ + rapl_update_domain_data(rp); /* first we register package domain as the parent zone*/ for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { @@ -1257,8 +1229,7 @@ static int rapl_package_register_powercap(struct rapl_package *rp) if (IS_ERR(power_zone)) { pr_debug("failed to register package, %d\n", rp->id); - ret = PTR_ERR(power_zone); - goto exit_package; + return PTR_ERR(power_zone); } /* track parent zone in per package/socket data */ rp->power_zone = power_zone; @@ -1268,8 +1239,7 @@ static int rapl_package_register_powercap(struct rapl_package *rp) } if (!power_zone) { pr_err("no package domain found, unknown topology!\n"); - ret = -ENODEV; - goto exit_package; + return -ENODEV; } /* now register domains as children of the socket/package*/ for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { @@ -1290,11 +1260,11 @@ static int rapl_package_register_powercap(struct rapl_package *rp) goto err_cleanup; } } + return 0; -exit_package: - return ret; err_cleanup: - /* clean up previously initialized domains within the package if we + /* + * Clean up previously initialized domains within the package if we * failed after the first domain setup. */ while (--rd >= rp->domains) { @@ -1305,7 +1275,7 @@ err_cleanup: return ret; } -static int rapl_register_psys(void) +static int __init rapl_register_psys(void) { struct rapl_domain *rd; struct powercap_zone *power_zone; @@ -1346,40 +1316,14 @@ static int rapl_register_psys(void) return 0; } -static int rapl_register_powercap(void) +static int __init rapl_register_powercap(void) { - struct rapl_domain *rd; - struct rapl_package *rp; - int ret = 0; - control_type = powercap_register_control_type(NULL, "intel-rapl", NULL); if (IS_ERR(control_type)) { pr_debug("failed to register powercap control_type.\n"); return PTR_ERR(control_type); } - /* read the initial data */ - rapl_update_domain_data(); - list_for_each_entry(rp, &rapl_packages, plist) - if (rapl_package_register_powercap(rp)) - goto err_cleanup_package; - - /* Don't bail out if PSys is not supported */ - rapl_register_psys(); - - return ret; - -err_cleanup_package: - /* clean up previously initialized packages */ - list_for_each_entry_continue_reverse(rp, &rapl_packages, plist) { - for (rd = rp->domains; rd < rp->domains + rp->nr_domains; - rd++) { - pr_debug("unregister zone/package %d, %s domain\n", - rp->id, rd->name); - powercap_unregister_zone(control_type, &rd->power_zone); - } - } - - return ret; + return 0; } static int rapl_check_domain(int cpu, int domain) @@ -1452,9 +1396,8 @@ static void rapl_detect_powerlimit(struct rapl_domain *rd) */ static int rapl_detect_domains(struct rapl_package *rp, int cpu) { - int i; - int ret = 0; struct rapl_domain *rd; + int i; for (i = 0; i < RAPL_DOMAIN_MAX; i++) { /* use physical package id to read counters */ @@ -1466,84 +1409,20 @@ static int rapl_detect_domains(struct rapl_package *rp, int cpu) rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX); if (!rp->nr_domains) { pr_debug("no valid rapl domains found in package %d\n", rp->id); - ret = -ENODEV; - goto done; + return -ENODEV; } pr_debug("found %d domains on package %d\n", rp->nr_domains, rp->id); rp->domains = kcalloc(rp->nr_domains + 1, sizeof(struct rapl_domain), GFP_KERNEL); - if (!rp->domains) { - ret = -ENOMEM; - goto done; - } + if (!rp->domains) + return -ENOMEM; + rapl_init_domains(rp); for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) rapl_detect_powerlimit(rd); - - -done: - return ret; -} - -static bool is_package_new(int package) -{ - struct rapl_package *rp; - - /* caller prevents cpu hotplug, there will be no new packages added - * or deleted while traversing the package list, no need for locking. - */ - list_for_each_entry(rp, &rapl_packages, plist) - if (package == rp->id) - return false; - - return true; -} - -/* RAPL interface can be made of a two-level hierarchy: package level and domain - * level. We first detect the number of packages then domains of each package. - * We have to consider the possiblity of CPU online/offline due to hotplug and - * other scenarios. - */ -static int rapl_detect_topology(void) -{ - int i; - int phy_package_id; - struct rapl_package *new_package, *rp; - - for_each_online_cpu(i) { - phy_package_id = topology_physical_package_id(i); - if (is_package_new(phy_package_id)) { - new_package = kzalloc(sizeof(*rp), GFP_KERNEL); - if (!new_package) { - rapl_cleanup_data(); - return -ENOMEM; - } - /* add the new package to the list */ - new_package->id = phy_package_id; - new_package->nr_cpus = 1; - /* use the first active cpu of the package to access */ - new_package->lead_cpu = i; - /* check if the package contains valid domains */ - if (rapl_detect_domains(new_package, i) || - rapl_defaults->check_unit(new_package, i)) { - kfree(new_package->domains); - kfree(new_package); - /* free up the packages already initialized */ - rapl_cleanup_data(); - return -ENODEV; - } - INIT_LIST_HEAD(&new_package->plist); - list_add(&new_package->plist, &rapl_packages); - } else { - rp = find_package_by_id(phy_package_id); - if (rp) - ++rp->nr_cpus; - } - } - return 0; } @@ -1552,12 +1431,21 @@ static void rapl_remove_package(struct rapl_package *rp) { struct rapl_domain *rd, *rd_package = NULL; + package_power_limit_irq_restore(rp); + for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) { + rapl_write_data_raw(rd, PL1_ENABLE, 0); + rapl_write_data_raw(rd, PL1_CLAMP, 0); + if (find_nr_power_limit(rd) > 1) { + rapl_write_data_raw(rd, PL2_ENABLE, 0); + rapl_write_data_raw(rd, PL2_CLAMP, 0); + } if (rd->id == RAPL_DOMAIN_PACKAGE) { rd_package = rd; continue; } - pr_debug("remove package %d, %s domain\n", rp->id, rd->name); + pr_debug("remove package, undo power limit on %d: %s\n", + rp->id, rd->name); powercap_unregister_zone(control_type, &rd->power_zone); } /* do parent zone last */ @@ -1567,20 +1455,17 @@ static void rapl_remove_package(struct rapl_package *rp) } /* called from CPU hotplug notifier, hotplug lock held */ -static int rapl_add_package(int cpu) +static struct rapl_package *rapl_add_package(int cpu, int pkgid) { - int ret = 0; - int phy_package_id; struct rapl_package *rp; + int ret; - phy_package_id = topology_physical_package_id(cpu); rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL); if (!rp) - return -ENOMEM; + return ERR_PTR(-ENOMEM); /* add the new package to the list */ - rp->id = phy_package_id; - rp->nr_cpus = 1; + rp->id = pkgid; rp->lead_cpu = cpu; /* check if the package contains valid domains */ @@ -1589,17 +1474,17 @@ static int rapl_add_package(int cpu) ret = -ENODEV; goto err_free_package; } - if (!rapl_package_register_powercap(rp)) { + ret = rapl_package_register_powercap(rp); + if (!ret) { INIT_LIST_HEAD(&rp->plist); list_add(&rp->plist, &rapl_packages); - return ret; + return rp; } err_free_package: kfree(rp->domains); kfree(rp); - - return ret; + return ERR_PTR(ret); } /* Handles CPU hotplug on multi-socket systems. @@ -1609,55 +1494,46 @@ err_free_package: * associated domains. Cooling devices are handled accordingly at * per-domain level. */ -static int rapl_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) +static int rapl_cpu_online(unsigned int cpu) { - unsigned long cpu = (unsigned long)hcpu; - int phy_package_id; + int pkgid = topology_physical_package_id(cpu); struct rapl_package *rp; - int lead_cpu; - phy_package_id = topology_physical_package_id(cpu); - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - case CPU_DOWN_FAILED: - case CPU_DOWN_FAILED_FROZEN: - rp = find_package_by_id(phy_package_id); - if (rp) - ++rp->nr_cpus; - else - rapl_add_package(cpu); - break; - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - rp = find_package_by_id(phy_package_id); - if (!rp) - break; - if (--rp->nr_cpus == 0) - rapl_remove_package(rp); - else if (cpu == rp->lead_cpu) { - /* choose another active cpu in the package */ - lead_cpu = cpumask_any_but(topology_core_cpumask(cpu), cpu); - if (lead_cpu < nr_cpu_ids) - rp->lead_cpu = lead_cpu; - else /* should never go here */ - pr_err("no active cpu available for package %d\n", - phy_package_id); - } + rp = find_package_by_id(pkgid); + if (!rp) { + rp = rapl_add_package(cpu, pkgid); + if (IS_ERR(rp)) + return PTR_ERR(rp); } + cpumask_set_cpu(cpu, &rp->cpumask); + return 0; +} + +static int rapl_cpu_down_prep(unsigned int cpu) +{ + int pkgid = topology_physical_package_id(cpu); + struct rapl_package *rp; + int lead_cpu; + + rp = find_package_by_id(pkgid); + if (!rp) + return 0; - return NOTIFY_OK; + cpumask_clear_cpu(cpu, &rp->cpumask); + lead_cpu = cpumask_first(&rp->cpumask); + if (lead_cpu >= nr_cpu_ids) + rapl_remove_package(rp); + else if (rp->lead_cpu == cpu) + rp->lead_cpu = lead_cpu; + return 0; } -static struct notifier_block rapl_cpu_notifier = { - .notifier_call = rapl_cpu_callback, -}; +static enum cpuhp_state pcap_rapl_online; static int __init rapl_init(void) { - int ret = 0; const struct x86_cpu_id *id; + int ret; id = x86_match_cpu(rapl_ids); if (!id) { @@ -1669,36 +1545,29 @@ static int __init rapl_init(void) rapl_defaults = (struct rapl_defaults *)id->driver_data; - cpu_notifier_register_begin(); - - /* prevent CPU hotplug during detection */ - get_online_cpus(); - ret = rapl_detect_topology(); + ret = rapl_register_powercap(); if (ret) - goto done; + return ret; - if (rapl_register_powercap()) { - rapl_cleanup_data(); - ret = -ENODEV; - goto done; - } - __register_hotcpu_notifier(&rapl_cpu_notifier); -done: - put_online_cpus(); - cpu_notifier_register_done(); + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powercap/rapl:online", + rapl_cpu_online, rapl_cpu_down_prep); + if (ret < 0) + goto err_unreg; + pcap_rapl_online = ret; + + /* Don't bail out if PSys is not supported */ + rapl_register_psys(); + return 0; +err_unreg: + rapl_unregister_powercap(); return ret; } static void __exit rapl_exit(void) { - cpu_notifier_register_begin(); - get_online_cpus(); - __unregister_hotcpu_notifier(&rapl_cpu_notifier); + cpuhp_remove_state(pcap_rapl_online); rapl_unregister_powercap(); - rapl_cleanup_data(); - put_online_cpus(); - cpu_notifier_register_done(); } module_init(rapl_init); diff --git a/drivers/thermal/intel_powerclamp.c b/drivers/thermal/intel_powerclamp.c index 350cb5e22ff3..df64692e9e64 100644 --- a/drivers/thermal/intel_powerclamp.c +++ b/drivers/thermal/intel_powerclamp.c @@ -43,7 +43,6 @@ #include <linux/kernel.h> #include <linux/delay.h> #include <linux/kthread.h> -#include <linux/freezer.h> #include <linux/cpu.h> #include <linux/thermal.h> #include <linux/slab.h> @@ -85,11 +84,26 @@ static unsigned int control_cpu; /* The cpu assigned to collect stat and update */ static bool clamping; +static const struct sched_param sparam = { + .sched_priority = MAX_USER_RT_PRIO / 2, +}; +struct powerclamp_worker_data { + struct kthread_worker *worker; + struct kthread_work balancing_work; + struct kthread_delayed_work idle_injection_work; + unsigned int cpu; + unsigned int count; + unsigned int guard; + unsigned int window_size_now; + unsigned int target_ratio; + unsigned int duration_jiffies; + bool clamping; +}; -static struct task_struct * __percpu *powerclamp_thread; +static struct powerclamp_worker_data * __percpu worker_data; static struct thermal_cooling_device *cooling_dev; static unsigned long *cpu_clamping_mask; /* bit map for tracking per cpu - * clamping thread + * clamping kthread worker */ static unsigned int duration; @@ -261,11 +275,6 @@ static u64 pkg_state_counter(void) return count; } -static void noop_timer(unsigned long foo) -{ - /* empty... just the fact that we get the interrupt wakes us up */ -} - static unsigned int get_compensation(int ratio) { unsigned int comp = 0; @@ -367,103 +376,79 @@ static bool powerclamp_adjust_controls(unsigned int target_ratio, return set_target_ratio + guard <= current_ratio; } -static int clamp_thread(void *arg) +static void clamp_balancing_func(struct kthread_work *work) { - int cpunr = (unsigned long)arg; - DEFINE_TIMER(wakeup_timer, noop_timer, 0, 0); - static const struct sched_param param = { - .sched_priority = MAX_USER_RT_PRIO/2, - }; - unsigned int count = 0; - unsigned int target_ratio; + struct powerclamp_worker_data *w_data; + int sleeptime; + unsigned long target_jiffies; + unsigned int compensated_ratio; + int interval; /* jiffies to sleep for each attempt */ - set_bit(cpunr, cpu_clamping_mask); - set_freezable(); - init_timer_on_stack(&wakeup_timer); - sched_setscheduler(current, SCHED_FIFO, ¶m); - - while (true == clamping && !kthread_should_stop() && - cpu_online(cpunr)) { - int sleeptime; - unsigned long target_jiffies; - unsigned int guard; - unsigned int compensated_ratio; - int interval; /* jiffies to sleep for each attempt */ - unsigned int duration_jiffies = msecs_to_jiffies(duration); - unsigned int window_size_now; - - try_to_freeze(); - /* - * make sure user selected ratio does not take effect until - * the next round. adjust target_ratio if user has changed - * target such that we can converge quickly. - */ - target_ratio = set_target_ratio; - guard = 1 + target_ratio/20; - window_size_now = window_size; - count++; - - /* - * systems may have different ability to enter package level - * c-states, thus we need to compensate the injected idle ratio - * to achieve the actual target reported by the HW. - */ - compensated_ratio = target_ratio + - get_compensation(target_ratio); - if (compensated_ratio <= 0) - compensated_ratio = 1; - interval = duration_jiffies * 100 / compensated_ratio; - - /* align idle time */ - target_jiffies = roundup(jiffies, interval); - sleeptime = target_jiffies - jiffies; - if (sleeptime <= 0) - sleeptime = 1; - schedule_timeout_interruptible(sleeptime); - /* - * only elected controlling cpu can collect stats and update - * control parameters. - */ - if (cpunr == control_cpu && !(count%window_size_now)) { - should_skip = - powerclamp_adjust_controls(target_ratio, - guard, window_size_now); - smp_mb(); - } + w_data = container_of(work, struct powerclamp_worker_data, + balancing_work); - if (should_skip) - continue; - - target_jiffies = jiffies + duration_jiffies; - mod_timer(&wakeup_timer, target_jiffies); - if (unlikely(local_softirq_pending())) - continue; - /* - * stop tick sched during idle time, interrupts are still - * allowed. thus jiffies are updated properly. - */ - preempt_disable(); - /* mwait until target jiffies is reached */ - while (time_before(jiffies, target_jiffies)) { - unsigned long ecx = 1; - unsigned long eax = target_mwait; - - /* - * REVISIT: may call enter_idle() to notify drivers who - * can save power during cpu idle. same for exit_idle() - */ - local_touch_nmi(); - stop_critical_timings(); - mwait_idle_with_hints(eax, ecx); - start_critical_timings(); - atomic_inc(&idle_wakeup_counter); - } - preempt_enable(); + /* + * make sure user selected ratio does not take effect until + * the next round. adjust target_ratio if user has changed + * target such that we can converge quickly. + */ + w_data->target_ratio = READ_ONCE(set_target_ratio); + w_data->guard = 1 + w_data->target_ratio / 20; + w_data->window_size_now = window_size; + w_data->duration_jiffies = msecs_to_jiffies(duration); + w_data->count++; + + /* + * systems may have different ability to enter package level + * c-states, thus we need to compensate the injected idle ratio + * to achieve the actual target reported by the HW. + */ + compensated_ratio = w_data->target_ratio + + get_compensation(w_data->target_ratio); + if (compensated_ratio <= 0) + compensated_ratio = 1; + interval = w_data->duration_jiffies * 100 / compensated_ratio; + + /* align idle time */ + target_jiffies = roundup(jiffies, interval); + sleeptime = target_jiffies - jiffies; + if (sleeptime <= 0) + sleeptime = 1; + + if (clamping && w_data->clamping && cpu_online(w_data->cpu)) + kthread_queue_delayed_work(w_data->worker, + &w_data->idle_injection_work, + sleeptime); +} + +static void clamp_idle_injection_func(struct kthread_work *work) +{ + struct powerclamp_worker_data *w_data; + + w_data = container_of(work, struct powerclamp_worker_data, + idle_injection_work.work); + + /* + * only elected controlling cpu can collect stats and update + * control parameters. + */ + if (w_data->cpu == control_cpu && + !(w_data->count % w_data->window_size_now)) { + should_skip = + powerclamp_adjust_controls(w_data->target_ratio, + w_data->guard, + w_data->window_size_now); + smp_mb(); } - del_timer_sync(&wakeup_timer); - clear_bit(cpunr, cpu_clamping_mask); - return 0; + if (should_skip) + goto balance; + + play_idle(jiffies_to_msecs(w_data->duration_jiffies)); + +balance: + if (clamping && w_data->clamping && cpu_online(w_data->cpu)) + kthread_queue_work(w_data->worker, &w_data->balancing_work); } /* @@ -507,10 +492,60 @@ static void poll_pkg_cstate(struct work_struct *dummy) schedule_delayed_work(&poll_pkg_cstate_work, HZ); } +static void start_power_clamp_worker(unsigned long cpu) +{ + struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu); + struct kthread_worker *worker; + + worker = kthread_create_worker_on_cpu(cpu, 0, "kidle_inject/%ld", cpu); + if (IS_ERR(worker)) + return; + + w_data->worker = worker; + w_data->count = 0; + w_data->cpu = cpu; + w_data->clamping = true; + set_bit(cpu, cpu_clamping_mask); + sched_setscheduler(worker->task, SCHED_FIFO, &sparam); + kthread_init_work(&w_data->balancing_work, clamp_balancing_func); + kthread_init_delayed_work(&w_data->idle_injection_work, + clamp_idle_injection_func); + kthread_queue_work(w_data->worker, &w_data->balancing_work); +} + +static void stop_power_clamp_worker(unsigned long cpu) +{ + struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu); + + if (!w_data->worker) + return; + + w_data->clamping = false; + /* + * Make sure that all works that get queued after this point see + * the clamping disabled. The counter part is not needed because + * there is an implicit memory barrier when the queued work + * is proceed. + */ + smp_wmb(); + kthread_cancel_work_sync(&w_data->balancing_work); + kthread_cancel_delayed_work_sync(&w_data->idle_injection_work); + /* + * The balancing work still might be queued here because + * the handling of the "clapming" variable, cancel, and queue + * operations are not synchronized via a lock. But it is not + * a big deal. The balancing work is fast and destroy kthread + * will wait for it. + */ + clear_bit(w_data->cpu, cpu_clamping_mask); + kthread_destroy_worker(w_data->worker); + + w_data->worker = NULL; +} + static int start_power_clamp(void) { unsigned long cpu; - struct task_struct *thread; set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1); /* prevent cpu hotplug */ @@ -524,22 +559,9 @@ static int start_power_clamp(void) clamping = true; schedule_delayed_work(&poll_pkg_cstate_work, 0); - /* start one thread per online cpu */ + /* start one kthread worker per online cpu */ for_each_online_cpu(cpu) { - struct task_struct **p = - per_cpu_ptr(powerclamp_thread, cpu); - - thread = kthread_create_on_node(clamp_thread, - (void *) cpu, - cpu_to_node(cpu), - "kidle_inject/%ld", cpu); - /* bind to cpu here */ - if (likely(!IS_ERR(thread))) { - kthread_bind(thread, cpu); - wake_up_process(thread); - *p = thread; - } - + start_power_clamp_worker(cpu); } put_online_cpus(); @@ -549,71 +571,49 @@ static int start_power_clamp(void) static void end_power_clamp(void) { int i; - struct task_struct *thread; - clamping = false; /* - * make clamping visible to other cpus and give per cpu clamping threads - * sometime to exit, or gets killed later. + * Block requeuing in all the kthread workers. They will flush and + * stop faster. */ - smp_mb(); - msleep(20); + clamping = false; if (bitmap_weight(cpu_clamping_mask, num_possible_cpus())) { for_each_set_bit(i, cpu_clamping_mask, num_possible_cpus()) { - pr_debug("clamping thread for cpu %d alive, kill\n", i); - thread = *per_cpu_ptr(powerclamp_thread, i); - kthread_stop(thread); + pr_debug("clamping worker for cpu %d alive, destroy\n", + i); + stop_power_clamp_worker(i); } } } -static int powerclamp_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) +static int powerclamp_cpu_online(unsigned int cpu) { - unsigned long cpu = (unsigned long)hcpu; - struct task_struct *thread; - struct task_struct **percpu_thread = - per_cpu_ptr(powerclamp_thread, cpu); - - if (false == clamping) - goto exit_ok; - - switch (action) { - case CPU_ONLINE: - thread = kthread_create_on_node(clamp_thread, - (void *) cpu, - cpu_to_node(cpu), - "kidle_inject/%lu", cpu); - if (likely(!IS_ERR(thread))) { - kthread_bind(thread, cpu); - wake_up_process(thread); - *percpu_thread = thread; - } - /* prefer BSP as controlling CPU */ - if (cpu == 0) { - control_cpu = 0; - smp_mb(); - } - break; - case CPU_DEAD: - if (test_bit(cpu, cpu_clamping_mask)) { - pr_err("cpu %lu dead but powerclamping thread is not\n", - cpu); - kthread_stop(*percpu_thread); - } - if (cpu == control_cpu) { - control_cpu = smp_processor_id(); - smp_mb(); - } + if (clamping == false) + return 0; + start_power_clamp_worker(cpu); + /* prefer BSP as controlling CPU */ + if (cpu == 0) { + control_cpu = 0; + smp_mb(); } - -exit_ok: - return NOTIFY_OK; + return 0; } -static struct notifier_block powerclamp_cpu_notifier = { - .notifier_call = powerclamp_cpu_callback, -}; +static int powerclamp_cpu_predown(unsigned int cpu) +{ + if (clamping == false) + return 0; + + stop_power_clamp_worker(cpu); + if (cpu != control_cpu) + return 0; + + control_cpu = cpumask_first(cpu_online_mask); + if (control_cpu == cpu) + control_cpu = cpumask_next(cpu, cpu_online_mask); + smp_mb(); + return 0; +} static int powerclamp_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state) @@ -741,6 +741,8 @@ file_error: debugfs_remove_recursive(debug_dir); } +static enum cpuhp_state hp_state; + static int __init powerclamp_init(void) { int retval; @@ -758,10 +760,17 @@ static int __init powerclamp_init(void) /* set default limit, maybe adjusted during runtime based on feedback */ window_size = 2; - register_hotcpu_notifier(&powerclamp_cpu_notifier); + retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "thermal/intel_powerclamp:online", + powerclamp_cpu_online, + powerclamp_cpu_predown); + if (retval < 0) + goto exit_free; + + hp_state = retval; - powerclamp_thread = alloc_percpu(struct task_struct *); - if (!powerclamp_thread) { + worker_data = alloc_percpu(struct powerclamp_worker_data); + if (!worker_data) { retval = -ENOMEM; goto exit_unregister; } @@ -781,9 +790,9 @@ static int __init powerclamp_init(void) return 0; exit_free_thread: - free_percpu(powerclamp_thread); + free_percpu(worker_data); exit_unregister: - unregister_hotcpu_notifier(&powerclamp_cpu_notifier); + cpuhp_remove_state_nocalls(hp_state); exit_free: kfree(cpu_clamping_mask); return retval; @@ -792,9 +801,9 @@ module_init(powerclamp_init); static void __exit powerclamp_exit(void) { - unregister_hotcpu_notifier(&powerclamp_cpu_notifier); end_power_clamp(); - free_percpu(powerclamp_thread); + cpuhp_remove_state_nocalls(hp_state); + free_percpu(worker_data); thermal_cooling_device_unregister(cooling_dev); kfree(cpu_clamping_mask); diff --git a/include/acpi/processor.h b/include/acpi/processor.h index f3db11c24654..c1ba00fc4888 100644 --- a/include/acpi/processor.h +++ b/include/acpi/processor.h @@ -249,6 +249,7 @@ extern int acpi_processor_register_performance(struct acpi_processor_performance *performance, unsigned int cpu); extern void acpi_processor_unregister_performance(unsigned int cpu); +int acpi_processor_pstate_control(void); /* note: this locks both the calling module and the processor module if a _PPC object exists, rmmod is disallowed then */ int acpi_processor_notify_smm(struct module *calling_module); @@ -294,7 +295,7 @@ static inline void acpi_processor_ffh_cstate_enter(struct acpi_processor_cx #ifdef CONFIG_CPU_FREQ void acpi_processor_ppc_init(void); void acpi_processor_ppc_exit(void); -int acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag); +void acpi_processor_ppc_has_changed(struct acpi_processor *pr, int event_flag); extern int acpi_processor_get_bios_limit(int cpu, unsigned int *limit); #else static inline void acpi_processor_ppc_init(void) diff --git a/include/linux/cpu.h b/include/linux/cpu.h index e571128ad99a..09807c2ce328 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -238,6 +238,8 @@ void arch_cpu_idle_dead(void); int cpu_report_state(int cpu); int cpu_check_up_prepare(int cpu); void cpu_set_state_online(int cpu); +void play_idle(unsigned long duration_ms); + #ifdef CONFIG_HOTPLUG_CPU bool cpu_wait_death(unsigned int cpu, int seconds); bool cpu_report_death(void); diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index 32dc0cbd51ca..7e05c5e4e45c 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -175,7 +175,7 @@ void disable_cpufreq(void); u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy); int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu); -int cpufreq_update_policy(unsigned int cpu); +void cpufreq_update_policy(unsigned int cpu); bool have_governor_per_policy(void); struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy); void cpufreq_enable_fast_switch(struct cpufreq_policy *policy); @@ -234,6 +234,10 @@ __ATTR(_name, _perm, show_##_name, NULL) static struct freq_attr _name = \ __ATTR(_name, 0644, show_##_name, store_##_name) +#define cpufreq_freq_attr_wo(_name) \ +static struct freq_attr _name = \ +__ATTR(_name, 0200, NULL, store_##_name) + struct global_attr { struct attribute attr; ssize_t (*show)(struct kobject *kobj, diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h index bb31373c3478..da346f2817a8 100644 --- a/include/linux/cpuidle.h +++ b/include/linux/cpuidle.h @@ -74,6 +74,7 @@ struct cpuidle_driver_kobj; struct cpuidle_device { unsigned int registered:1; unsigned int enabled:1; + unsigned int use_deepest_state:1; unsigned int cpu; int last_residency; @@ -192,11 +193,12 @@ static inline struct cpuidle_driver *cpuidle_get_cpu_driver( static inline struct cpuidle_device *cpuidle_get_device(void) {return NULL; } #endif -#if defined(CONFIG_CPU_IDLE) && defined(CONFIG_SUSPEND) +#ifdef CONFIG_CPU_IDLE extern int cpuidle_find_deepest_state(struct cpuidle_driver *drv, struct cpuidle_device *dev); extern int cpuidle_enter_freeze(struct cpuidle_driver *drv, struct cpuidle_device *dev); +extern void cpuidle_use_deepest_state(bool enable); #else static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv, struct cpuidle_device *dev) @@ -204,6 +206,9 @@ static inline int cpuidle_find_deepest_state(struct cpuidle_driver *drv, static inline int cpuidle_enter_freeze(struct cpuidle_driver *drv, struct cpuidle_device *dev) {return -ENODEV; } +static inline void cpuidle_use_deepest_state(bool enable) +{ +} #endif /* kernel/sched/idle.c */ @@ -235,8 +240,6 @@ struct cpuidle_governor { int (*select) (struct cpuidle_driver *drv, struct cpuidle_device *dev); void (*reflect) (struct cpuidle_device *dev, int index); - - struct module *owner; }; #ifdef CONFIG_CPU_IDLE diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index a09fe5c009c8..81ece61075df 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -15,11 +15,11 @@ #include <linux/err.h> #include <linux/of.h> #include <linux/notifier.h> +#include <linux/spinlock.h> /* Defines used for the flags field in the struct generic_pm_domain */ #define GENPD_FLAG_PM_CLK (1U << 0) /* PM domain uses PM clk */ - -#define GENPD_MAX_NUM_STATES 8 /* Number of possible low power states */ +#define GENPD_FLAG_IRQ_SAFE (1U << 1) /* PM domain operates in atomic */ enum gpd_status { GPD_STATE_ACTIVE = 0, /* PM domain is active */ @@ -40,15 +40,18 @@ struct gpd_dev_ops { struct genpd_power_state { s64 power_off_latency_ns; s64 power_on_latency_ns; + s64 residency_ns; + struct fwnode_handle *fwnode; }; +struct genpd_lock_ops; + struct generic_pm_domain { struct dev_pm_domain domain; /* PM domain operations */ struct list_head gpd_list_node; /* Node in the global PM domains list */ struct list_head master_links; /* Links with PM domain as a master */ struct list_head slave_links; /* Links with PM domain as a slave */ struct list_head dev_list; /* List of devices */ - struct mutex lock; struct dev_power_governor *gov; struct work_struct power_off_work; struct fwnode_handle *provider; /* Identity of the domain provider */ @@ -70,9 +73,18 @@ struct generic_pm_domain { void (*detach_dev)(struct generic_pm_domain *domain, struct device *dev); unsigned int flags; /* Bit field of configs for genpd */ - struct genpd_power_state states[GENPD_MAX_NUM_STATES]; + struct genpd_power_state *states; unsigned int state_count; /* number of states */ unsigned int state_idx; /* state that genpd will go to when off */ + void *free; /* Free the state that was allocated for default */ + const struct genpd_lock_ops *lock_ops; + union { + struct mutex mlock; + struct { + spinlock_t slock; + unsigned long lock_flags; + }; + }; }; @@ -205,6 +217,8 @@ extern int of_genpd_add_device(struct of_phandle_args *args, extern int of_genpd_add_subdomain(struct of_phandle_args *parent, struct of_phandle_args *new_subdomain); extern struct generic_pm_domain *of_genpd_remove_last(struct device_node *np); +extern int of_genpd_parse_idle_states(struct device_node *dn, + struct genpd_power_state **states, int *n); int genpd_dev_pm_attach(struct device *dev); #else /* !CONFIG_PM_GENERIC_DOMAINS_OF */ @@ -234,6 +248,12 @@ static inline int of_genpd_add_subdomain(struct of_phandle_args *parent, return -ENODEV; } +static inline int of_genpd_parse_idle_states(struct device_node *dn, + struct genpd_power_state **states, int *n) +{ + return -ENODEV; +} + static inline int genpd_dev_pm_attach(struct device *dev) { return -ENODEV; diff --git a/include/linux/pm_opp.h b/include/linux/pm_opp.h index bca26157f5b6..0edd88f93904 100644 --- a/include/linux/pm_opp.h +++ b/include/linux/pm_opp.h @@ -17,13 +17,65 @@ #include <linux/err.h> #include <linux/notifier.h> +struct clk; +struct regulator; struct dev_pm_opp; struct device; +struct opp_table; enum dev_pm_opp_event { OPP_EVENT_ADD, OPP_EVENT_REMOVE, OPP_EVENT_ENABLE, OPP_EVENT_DISABLE, }; +/** + * struct dev_pm_opp_supply - Power supply voltage/current values + * @u_volt: Target voltage in microvolts corresponding to this OPP + * @u_volt_min: Minimum voltage in microvolts corresponding to this OPP + * @u_volt_max: Maximum voltage in microvolts corresponding to this OPP + * @u_amp: Maximum current drawn by the device in microamperes + * + * This structure stores the voltage/current values for a single power supply. + */ +struct dev_pm_opp_supply { + unsigned long u_volt; + unsigned long u_volt_min; + unsigned long u_volt_max; + unsigned long u_amp; +}; + +/** + * struct dev_pm_opp_info - OPP freq/voltage/current values + * @rate: Target clk rate in hz + * @supplies: Array of voltage/current values for all power supplies + * + * This structure stores the freq/voltage/current values for a single OPP. + */ +struct dev_pm_opp_info { + unsigned long rate; + struct dev_pm_opp_supply *supplies; +}; + +/** + * struct dev_pm_set_opp_data - Set OPP data + * @old_opp: Old OPP info + * @new_opp: New OPP info + * @regulators: Array of regulator pointers + * @regulator_count: Number of regulators + * @clk: Pointer to clk + * @dev: Pointer to the struct device + * + * This structure contains all information required for setting an OPP. + */ +struct dev_pm_set_opp_data { + struct dev_pm_opp_info old_opp; + struct dev_pm_opp_info new_opp; + + struct regulator **regulators; + unsigned int regulator_count; + struct clk *clk; + struct device *dev; +}; + #if defined(CONFIG_PM_OPP) unsigned long dev_pm_opp_get_voltage(struct dev_pm_opp *opp); @@ -62,8 +114,10 @@ int dev_pm_opp_set_supported_hw(struct device *dev, const u32 *versions, void dev_pm_opp_put_supported_hw(struct device *dev); int dev_pm_opp_set_prop_name(struct device *dev, const char *name); void dev_pm_opp_put_prop_name(struct device *dev); -int dev_pm_opp_set_regulator(struct device *dev, const char *name); -void dev_pm_opp_put_regulator(struct device *dev); +struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[], unsigned int count); +void dev_pm_opp_put_regulators(struct opp_table *opp_table); +int dev_pm_opp_register_set_opp_helper(struct device *dev, int (*set_opp)(struct dev_pm_set_opp_data *data)); +void dev_pm_opp_register_put_opp_helper(struct device *dev); int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq); int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, const struct cpumask *cpumask); int dev_pm_opp_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask); @@ -163,6 +217,14 @@ static inline int dev_pm_opp_set_supported_hw(struct device *dev, static inline void dev_pm_opp_put_supported_hw(struct device *dev) {} +static inline int dev_pm_opp_register_set_opp_helper(struct device *dev, + int (*set_opp)(struct dev_pm_set_opp_data *data)) +{ + return -ENOTSUPP; +} + +static inline void dev_pm_opp_register_put_opp_helper(struct device *dev) {} + static inline int dev_pm_opp_set_prop_name(struct device *dev, const char *name) { return -ENOTSUPP; @@ -170,12 +232,12 @@ static inline int dev_pm_opp_set_prop_name(struct device *dev, const char *name) static inline void dev_pm_opp_put_prop_name(struct device *dev) {} -static inline int dev_pm_opp_set_regulator(struct device *dev, const char *name) +static inline struct opp_table *dev_pm_opp_set_regulators(struct device *dev, const char * const names[], unsigned int count) { - return -ENOTSUPP; + return ERR_PTR(-ENOTSUPP); } -static inline void dev_pm_opp_put_regulator(struct device *dev) {} +static inline void dev_pm_opp_put_regulators(struct opp_table *opp_table) {} static inline int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) { diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 2e14d2667b6c..4957fc185ea9 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -61,12 +61,6 @@ static inline void pm_suspend_ignore_children(struct device *dev, bool enable) dev->power.ignore_children = enable; } -static inline bool pm_children_suspended(struct device *dev) -{ - return dev->power.ignore_children - || !atomic_read(&dev->power.child_count); -} - static inline void pm_runtime_get_noresume(struct device *dev) { atomic_inc(&dev->power.usage_count); @@ -162,7 +156,6 @@ static inline void pm_runtime_allow(struct device *dev) {} static inline void pm_runtime_forbid(struct device *dev) {} static inline void pm_suspend_ignore_children(struct device *dev, bool enable) {} -static inline bool pm_children_suspended(struct device *dev) { return false; } static inline void pm_runtime_get_noresume(struct device *dev) {} static inline void pm_runtime_put_noidle(struct device *dev) {} static inline bool device_run_wake(struct device *dev) { return false; } @@ -265,9 +258,9 @@ static inline int pm_runtime_set_active(struct device *dev) return __pm_runtime_set_status(dev, RPM_ACTIVE); } -static inline void pm_runtime_set_suspended(struct device *dev) +static inline int pm_runtime_set_suspended(struct device *dev) { - __pm_runtime_set_status(dev, RPM_SUSPENDED); + return __pm_runtime_set_status(dev, RPM_SUSPENDED); } static inline void pm_runtime_disable(struct device *dev) diff --git a/include/linux/sched.h b/include/linux/sched.h index 0e90f2973719..5ccbbfe41345 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2287,6 +2287,7 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, /* * Per process flags */ +#define PF_IDLE 0x00000002 /* I am an IDLE thread */ #define PF_EXITING 0x00000004 /* getting shut down */ #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ @@ -2648,7 +2649,7 @@ extern struct task_struct *idle_task(int cpu); */ static inline bool is_idle_task(const struct task_struct *p) { - return p->pid == 0; + return !!(p->flags & PF_IDLE); } extern struct task_struct *curr_task(int cpu); extern void ia64_set_curr_task(int cpu, struct task_struct *p); diff --git a/include/linux/suspend.h b/include/linux/suspend.h index d9718378a8be..0c729c3c8549 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -194,6 +194,8 @@ struct platform_freeze_ops { }; #ifdef CONFIG_SUSPEND +extern suspend_state_t mem_sleep_default; + /** * suspend_set_ops - set platform dependent suspend operations * @ops: The new suspend operations to set. diff --git a/kernel/fork.c b/kernel/fork.c index 7377f414f3ce..a439ac429669 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1544,7 +1544,7 @@ static __latent_entropy struct task_struct *copy_process( goto bad_fork_cleanup_count; delayacct_tsk_init(p); /* Must remain after dup_task_struct() */ - p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER); + p->flags &= ~(PF_SUPERPRIV | PF_WQ_WORKER | PF_IDLE); p->flags |= PF_FORKNOEXEC; INIT_LIST_HEAD(&p->children); INIT_LIST_HEAD(&p->sibling); diff --git a/kernel/power/main.c b/kernel/power/main.c index 281a697fd458..d401c21136d1 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -78,6 +78,78 @@ static ssize_t pm_async_store(struct kobject *kobj, struct kobj_attribute *attr, power_attr(pm_async); +#ifdef CONFIG_SUSPEND +static ssize_t mem_sleep_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + char *s = buf; + suspend_state_t i; + + for (i = PM_SUSPEND_MIN; i < PM_SUSPEND_MAX; i++) + if (mem_sleep_states[i]) { + const char *label = mem_sleep_states[i]; + + if (mem_sleep_current == i) + s += sprintf(s, "[%s] ", label); + else + s += sprintf(s, "%s ", label); + } + + /* Convert the last space to a newline if needed. */ + if (s != buf) + *(s-1) = '\n'; + + return (s - buf); +} + +static suspend_state_t decode_suspend_state(const char *buf, size_t n) +{ + suspend_state_t state; + char *p; + int len; + + p = memchr(buf, '\n', n); + len = p ? p - buf : n; + + for (state = PM_SUSPEND_MIN; state < PM_SUSPEND_MAX; state++) { + const char *label = mem_sleep_states[state]; + + if (label && len == strlen(label) && !strncmp(buf, label, len)) + return state; + } + + return PM_SUSPEND_ON; +} + +static ssize_t mem_sleep_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t n) +{ + suspend_state_t state; + int error; + + error = pm_autosleep_lock(); + if (error) + return error; + + if (pm_autosleep_state() > PM_SUSPEND_ON) { + error = -EBUSY; + goto out; + } + + state = decode_suspend_state(buf, n); + if (state < PM_SUSPEND_MAX && state > PM_SUSPEND_ON) + mem_sleep_current = state; + else + error = -EINVAL; + + out: + pm_autosleep_unlock(); + return error ? error : n; +} + +power_attr(mem_sleep); +#endif /* CONFIG_SUSPEND */ + #ifdef CONFIG_PM_DEBUG int pm_test_level = TEST_NONE; @@ -368,12 +440,16 @@ static ssize_t state_store(struct kobject *kobj, struct kobj_attribute *attr, } state = decode_state(buf, n); - if (state < PM_SUSPEND_MAX) + if (state < PM_SUSPEND_MAX) { + if (state == PM_SUSPEND_MEM) + state = mem_sleep_current; + error = pm_suspend(state); - else if (state == PM_SUSPEND_MAX) + } else if (state == PM_SUSPEND_MAX) { error = hibernate(); - else + } else { error = -EINVAL; + } out: pm_autosleep_unlock(); @@ -485,6 +561,9 @@ static ssize_t autosleep_store(struct kobject *kobj, && strcmp(buf, "off") && strcmp(buf, "off\n")) return -EINVAL; + if (state == PM_SUSPEND_MEM) + state = mem_sleep_current; + error = pm_autosleep_set_state(state); return error ? error : n; } @@ -602,6 +681,9 @@ static struct attribute * g[] = { #ifdef CONFIG_PM_SLEEP &pm_async_attr.attr, &wakeup_count_attr.attr, +#ifdef CONFIG_SUSPEND + &mem_sleep_attr.attr, +#endif #ifdef CONFIG_PM_AUTOSLEEP &autosleep_attr.attr, #endif diff --git a/kernel/power/power.h b/kernel/power/power.h index 56d1d0dedf76..1dfa0da827d3 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -189,11 +189,15 @@ extern void swsusp_show_speed(ktime_t, ktime_t, unsigned int, char *); #ifdef CONFIG_SUSPEND /* kernel/power/suspend.c */ -extern const char *pm_labels[]; +extern const char * const pm_labels[]; extern const char *pm_states[]; +extern const char *mem_sleep_states[]; +extern suspend_state_t mem_sleep_current; extern int suspend_devices_and_enter(suspend_state_t state); #else /* !CONFIG_SUSPEND */ +#define mem_sleep_current PM_SUSPEND_ON + static inline int suspend_devices_and_enter(suspend_state_t state) { return -ENOSYS; diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 6ccb08f57fcb..f67ceb7768b8 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -32,8 +32,21 @@ #include "power.h" -const char *pm_labels[] = { "mem", "standby", "freeze", NULL }; +const char * const pm_labels[] = { + [PM_SUSPEND_FREEZE] = "freeze", + [PM_SUSPEND_STANDBY] = "standby", + [PM_SUSPEND_MEM] = "mem", +}; const char *pm_states[PM_SUSPEND_MAX]; +static const char * const mem_sleep_labels[] = { + [PM_SUSPEND_FREEZE] = "s2idle", + [PM_SUSPEND_STANDBY] = "shallow", + [PM_SUSPEND_MEM] = "deep", +}; +const char *mem_sleep_states[PM_SUSPEND_MAX]; + +suspend_state_t mem_sleep_current = PM_SUSPEND_FREEZE; +suspend_state_t mem_sleep_default = PM_SUSPEND_MAX; unsigned int pm_suspend_global_flags; EXPORT_SYMBOL_GPL(pm_suspend_global_flags); @@ -110,30 +123,32 @@ static bool valid_state(suspend_state_t state) return suspend_ops && suspend_ops->valid && suspend_ops->valid(state); } -/* - * If this is set, the "mem" label always corresponds to the deepest sleep state - * available, the "standby" label corresponds to the second deepest sleep state - * available (if any), and the "freeze" label corresponds to the remaining - * available sleep state (if there is one). - */ -static bool relative_states; - void __init pm_states_init(void) { + /* "mem" and "freeze" are always present in /sys/power/state. */ + pm_states[PM_SUSPEND_MEM] = pm_labels[PM_SUSPEND_MEM]; + pm_states[PM_SUSPEND_FREEZE] = pm_labels[PM_SUSPEND_FREEZE]; /* - * freeze state should be supported even without any suspend_ops, - * initialize pm_states accordingly here + * Suspend-to-idle should be supported even without any suspend_ops, + * initialize mem_sleep_states[] accordingly here. */ - pm_states[PM_SUSPEND_FREEZE] = pm_labels[relative_states ? 0 : 2]; + mem_sleep_states[PM_SUSPEND_FREEZE] = mem_sleep_labels[PM_SUSPEND_FREEZE]; } -static int __init sleep_states_setup(char *str) +static int __init mem_sleep_default_setup(char *str) { - relative_states = !strncmp(str, "1", 1); + suspend_state_t state; + + for (state = PM_SUSPEND_FREEZE; state <= PM_SUSPEND_MEM; state++) + if (mem_sleep_labels[state] && + !strcmp(str, mem_sleep_labels[state])) { + mem_sleep_default = state; + break; + } + return 1; } - -__setup("relative_sleep_states=", sleep_states_setup); +__setup("mem_sleep_default=", mem_sleep_default_setup); /** * suspend_set_ops - Set the global suspend method table. @@ -141,21 +156,21 @@ __setup("relative_sleep_states=", sleep_states_setup); */ void suspend_set_ops(const struct platform_suspend_ops *ops) { - suspend_state_t i; - int j = 0; - lock_system_sleep(); suspend_ops = ops; - for (i = PM_SUSPEND_MEM; i >= PM_SUSPEND_STANDBY; i--) - if (valid_state(i)) { - pm_states[i] = pm_labels[j++]; - } else if (!relative_states) { - pm_states[i] = NULL; - j++; - } - pm_states[PM_SUSPEND_FREEZE] = pm_labels[j]; + if (valid_state(PM_SUSPEND_STANDBY)) { + mem_sleep_states[PM_SUSPEND_STANDBY] = mem_sleep_labels[PM_SUSPEND_STANDBY]; + pm_states[PM_SUSPEND_STANDBY] = pm_labels[PM_SUSPEND_STANDBY]; + if (mem_sleep_default == PM_SUSPEND_STANDBY) + mem_sleep_current = PM_SUSPEND_STANDBY; + } + if (valid_state(PM_SUSPEND_MEM)) { + mem_sleep_states[PM_SUSPEND_MEM] = mem_sleep_labels[PM_SUSPEND_MEM]; + if (mem_sleep_default >= PM_SUSPEND_MEM) + mem_sleep_current = PM_SUSPEND_MEM; + } unlock_system_sleep(); } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index d18804491d9f..966556ebdbb3 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -5280,6 +5280,7 @@ void init_idle(struct task_struct *idle, int cpu) __sched_fork(0, idle); idle->state = TASK_RUNNING; idle->se.exec_start = sched_clock(); + idle->flags |= PF_IDLE; kasan_unpoison_task_stack(idle); diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 69e06898997d..fd4659313640 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -12,11 +12,14 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/cpufreq.h> +#include <linux/kthread.h> #include <linux/slab.h> #include <trace/events/power.h> #include "sched.h" +#define SUGOV_KTHREAD_PRIORITY 50 + struct sugov_tunables { struct gov_attr_set attr_set; unsigned int rate_limit_us; @@ -35,8 +38,10 @@ struct sugov_policy { /* The next fields are only needed if fast switch cannot be used. */ struct irq_work irq_work; - struct work_struct work; + struct kthread_work work; struct mutex work_lock; + struct kthread_worker worker; + struct task_struct *thread; bool work_in_progress; bool need_freq_update; @@ -291,7 +296,7 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time, raw_spin_unlock(&sg_policy->update_lock); } -static void sugov_work(struct work_struct *work) +static void sugov_work(struct kthread_work *work) { struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work); @@ -308,7 +313,21 @@ static void sugov_irq_work(struct irq_work *irq_work) struct sugov_policy *sg_policy; sg_policy = container_of(irq_work, struct sugov_policy, irq_work); - schedule_work_on(smp_processor_id(), &sg_policy->work); + + /* + * For RT and deadline tasks, the schedutil governor shoots the + * frequency to maximum. Special care must be taken to ensure that this + * kthread doesn't result in the same behavior. + * + * This is (mostly) guaranteed by the work_in_progress flag. The flag is + * updated only at the end of the sugov_work() function and before that + * the schedutil governor rejects all other frequency scaling requests. + * + * There is a very rare case though, where the RT thread yields right + * after the work_in_progress flag is cleared. The effects of that are + * neglected for now. + */ + kthread_queue_work(&sg_policy->worker, &sg_policy->work); } /************************** sysfs interface ************************/ @@ -371,19 +390,64 @@ static struct sugov_policy *sugov_policy_alloc(struct cpufreq_policy *policy) return NULL; sg_policy->policy = policy; - init_irq_work(&sg_policy->irq_work, sugov_irq_work); - INIT_WORK(&sg_policy->work, sugov_work); - mutex_init(&sg_policy->work_lock); raw_spin_lock_init(&sg_policy->update_lock); return sg_policy; } static void sugov_policy_free(struct sugov_policy *sg_policy) { - mutex_destroy(&sg_policy->work_lock); kfree(sg_policy); } +static int sugov_kthread_create(struct sugov_policy *sg_policy) +{ + struct task_struct *thread; + struct sched_param param = { .sched_priority = MAX_USER_RT_PRIO / 2 }; + struct cpufreq_policy *policy = sg_policy->policy; + int ret; + + /* kthread only required for slow path */ + if (policy->fast_switch_enabled) + return 0; + + kthread_init_work(&sg_policy->work, sugov_work); + kthread_init_worker(&sg_policy->worker); + thread = kthread_create(kthread_worker_fn, &sg_policy->worker, + "sugov:%d", + cpumask_first(policy->related_cpus)); + if (IS_ERR(thread)) { + pr_err("failed to create sugov thread: %ld\n", PTR_ERR(thread)); + return PTR_ERR(thread); + } + + ret = sched_setscheduler_nocheck(thread, SCHED_FIFO, ¶m); + if (ret) { + kthread_stop(thread); + pr_warn("%s: failed to set SCHED_FIFO\n", __func__); + return ret; + } + + sg_policy->thread = thread; + kthread_bind_mask(thread, policy->related_cpus); + init_irq_work(&sg_policy->irq_work, sugov_irq_work); + mutex_init(&sg_policy->work_lock); + + wake_up_process(thread); + + return 0; +} + +static void sugov_kthread_stop(struct sugov_policy *sg_policy) +{ + /* kthread only required for slow path */ + if (sg_policy->policy->fast_switch_enabled) + return; + + kthread_flush_worker(&sg_policy->worker); + kthread_stop(sg_policy->thread); + mutex_destroy(&sg_policy->work_lock); +} + static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy) { struct sugov_tunables *tunables; @@ -416,16 +480,24 @@ static int sugov_init(struct cpufreq_policy *policy) if (policy->governor_data) return -EBUSY; + cpufreq_enable_fast_switch(policy); + sg_policy = sugov_policy_alloc(policy); - if (!sg_policy) - return -ENOMEM; + if (!sg_policy) { + ret = -ENOMEM; + goto disable_fast_switch; + } + + ret = sugov_kthread_create(sg_policy); + if (ret) + goto free_sg_policy; mutex_lock(&global_tunables_lock); if (global_tunables) { if (WARN_ON(have_governor_per_policy())) { ret = -EINVAL; - goto free_sg_policy; + goto stop_kthread; } policy->governor_data = sg_policy; sg_policy->tunables = global_tunables; @@ -437,7 +509,7 @@ static int sugov_init(struct cpufreq_policy *policy) tunables = sugov_tunables_alloc(sg_policy); if (!tunables) { ret = -ENOMEM; - goto free_sg_policy; + goto stop_kthread; } tunables->rate_limit_us = LATENCY_MULTIPLIER; @@ -454,20 +526,25 @@ static int sugov_init(struct cpufreq_policy *policy) if (ret) goto fail; - out: +out: mutex_unlock(&global_tunables_lock); - - cpufreq_enable_fast_switch(policy); return 0; - fail: +fail: policy->governor_data = NULL; sugov_tunables_free(tunables); - free_sg_policy: +stop_kthread: + sugov_kthread_stop(sg_policy); + +free_sg_policy: mutex_unlock(&global_tunables_lock); sugov_policy_free(sg_policy); + +disable_fast_switch: + cpufreq_disable_fast_switch(policy); + pr_err("initialization failed (error %d)\n", ret); return ret; } @@ -478,8 +555,6 @@ static void sugov_exit(struct cpufreq_policy *policy) struct sugov_tunables *tunables = sg_policy->tunables; unsigned int count; - cpufreq_disable_fast_switch(policy); - mutex_lock(&global_tunables_lock); count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook); @@ -489,7 +564,9 @@ static void sugov_exit(struct cpufreq_policy *policy) mutex_unlock(&global_tunables_lock); + sugov_kthread_stop(sg_policy); sugov_policy_free(sg_policy); + cpufreq_disable_fast_switch(policy); } static int sugov_start(struct cpufreq_policy *policy) @@ -535,8 +612,10 @@ static void sugov_stop(struct cpufreq_policy *policy) synchronize_sched(); - irq_work_sync(&sg_policy->irq_work); - cancel_work_sync(&sg_policy->work); + if (!policy->fast_switch_enabled) { + irq_work_sync(&sg_policy->irq_work); + kthread_cancel_work_sync(&sg_policy->work); + } } static void sugov_limits(struct cpufreq_policy *policy) diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 1d8718d5300d..6a4bae0a649d 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -164,11 +164,14 @@ static void cpuidle_idle_call(void) * timekeeping to prevent timer interrupts from kicking us out of idle * until a proper wakeup interrupt happens. */ - if (idle_should_freeze()) { - entered_state = cpuidle_enter_freeze(drv, dev); - if (entered_state > 0) { - local_irq_enable(); - goto exit_idle; + + if (idle_should_freeze() || dev->use_deepest_state) { + if (idle_should_freeze()) { + entered_state = cpuidle_enter_freeze(drv, dev); + if (entered_state > 0) { + local_irq_enable(); + goto exit_idle; + } } next_state = cpuidle_find_deepest_state(drv, dev); @@ -202,76 +205,65 @@ exit_idle: * * Called with polling cleared. */ -static void cpu_idle_loop(void) +static void do_idle(void) { - int cpu = smp_processor_id(); - - while (1) { - /* - * If the arch has a polling bit, we maintain an invariant: - * - * Our polling bit is clear if we're not scheduled (i.e. if - * rq->curr != rq->idle). This means that, if rq->idle has - * the polling bit set, then setting need_resched is - * guaranteed to cause the cpu to reschedule. - */ - - __current_set_polling(); - quiet_vmstat(); - tick_nohz_idle_enter(); + /* + * If the arch has a polling bit, we maintain an invariant: + * + * Our polling bit is clear if we're not scheduled (i.e. if rq->curr != + * rq->idle). This means that, if rq->idle has the polling bit set, + * then setting need_resched is guaranteed to cause the CPU to + * reschedule. + */ - while (!need_resched()) { - check_pgt_cache(); - rmb(); + __current_set_polling(); + tick_nohz_idle_enter(); - if (cpu_is_offline(cpu)) { - cpuhp_report_idle_dead(); - arch_cpu_idle_dead(); - } + while (!need_resched()) { + check_pgt_cache(); + rmb(); - local_irq_disable(); - arch_cpu_idle_enter(); - - /* - * In poll mode we reenable interrupts and spin. - * - * Also if we detected in the wakeup from idle - * path that the tick broadcast device expired - * for us, we don't want to go deep idle as we - * know that the IPI is going to arrive right - * away - */ - if (cpu_idle_force_poll || tick_check_broadcast_expired()) - cpu_idle_poll(); - else - cpuidle_idle_call(); - - arch_cpu_idle_exit(); + if (cpu_is_offline(smp_processor_id())) { + cpuhp_report_idle_dead(); + arch_cpu_idle_dead(); } - /* - * Since we fell out of the loop above, we know - * TIF_NEED_RESCHED must be set, propagate it into - * PREEMPT_NEED_RESCHED. - * - * This is required because for polling idle loops we will - * not have had an IPI to fold the state for us. - */ - preempt_set_need_resched(); - tick_nohz_idle_exit(); - __current_clr_polling(); + local_irq_disable(); + arch_cpu_idle_enter(); /* - * We promise to call sched_ttwu_pending and reschedule - * if need_resched is set while polling is set. That - * means that clearing polling needs to be visible - * before doing these things. + * In poll mode we reenable interrupts and spin. Also if we + * detected in the wakeup from idle path that the tick + * broadcast device expired for us, we don't want to go deep + * idle as we know that the IPI is going to arrive right away. */ - smp_mb__after_atomic(); - - sched_ttwu_pending(); - schedule_preempt_disabled(); + if (cpu_idle_force_poll || tick_check_broadcast_expired()) + cpu_idle_poll(); + else + cpuidle_idle_call(); + arch_cpu_idle_exit(); } + + /* + * Since we fell out of the loop above, we know TIF_NEED_RESCHED must + * be set, propagate it into PREEMPT_NEED_RESCHED. + * + * This is required because for polling idle loops we will not have had + * an IPI to fold the state for us. + */ + preempt_set_need_resched(); + tick_nohz_idle_exit(); + __current_clr_polling(); + + /* + * We promise to call sched_ttwu_pending() and reschedule if + * need_resched() is set while polling is set. That means that clearing + * polling needs to be visible before doing these things. + */ + smp_mb__after_atomic(); + + sched_ttwu_pending(); + schedule_preempt_disabled(); } bool cpu_in_idle(unsigned long pc) @@ -280,6 +272,56 @@ bool cpu_in_idle(unsigned long pc) pc < (unsigned long)__cpuidle_text_end; } +struct idle_timer { + struct hrtimer timer; + int done; +}; + +static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer) +{ + struct idle_timer *it = container_of(timer, struct idle_timer, timer); + + WRITE_ONCE(it->done, 1); + set_tsk_need_resched(current); + + return HRTIMER_NORESTART; +} + +void play_idle(unsigned long duration_ms) +{ + struct idle_timer it; + + /* + * Only FIFO tasks can disable the tick since they don't need the forced + * preemption. + */ + WARN_ON_ONCE(current->policy != SCHED_FIFO); + WARN_ON_ONCE(current->nr_cpus_allowed != 1); + WARN_ON_ONCE(!(current->flags & PF_KTHREAD)); + WARN_ON_ONCE(!(current->flags & PF_NO_SETAFFINITY)); + WARN_ON_ONCE(!duration_ms); + + rcu_sleep_check(); + preempt_disable(); + current->flags |= PF_IDLE; + cpuidle_use_deepest_state(true); + + it.done = 0; + hrtimer_init_on_stack(&it.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + it.timer.function = idle_inject_timer_fn; + hrtimer_start(&it.timer, ms_to_ktime(duration_ms), HRTIMER_MODE_REL_PINNED); + + while (!READ_ONCE(it.done)) + do_idle(); + + cpuidle_use_deepest_state(false); + current->flags &= ~PF_IDLE; + + preempt_fold_need_resched(); + preempt_enable(); +} +EXPORT_SYMBOL_GPL(play_idle); + void cpu_startup_entry(enum cpuhp_state state) { /* @@ -299,5 +341,6 @@ void cpu_startup_entry(enum cpuhp_state state) #endif arch_cpu_idle_prepare(); cpuhp_online_idle(state); - cpu_idle_loop(); + while (1) + do_idle(); } diff --git a/mm/kasan/kasan.c b/mm/kasan/kasan.c index 0e9505f66ec1..b2a0cff2bb35 100644 --- a/mm/kasan/kasan.c +++ b/mm/kasan/kasan.c @@ -80,7 +80,14 @@ void kasan_unpoison_task_stack(struct task_struct *task) /* Unpoison the stack for the current task beyond a watermark sp value. */ asmlinkage void kasan_unpoison_task_stack_below(const void *watermark) { - __kasan_unpoison_stack(current, watermark); + /* + * Calculate the task stack base address. Avoid using 'current' + * because this function is called by early resume code which hasn't + * yet set up the percpu register (%gs). + */ + void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1)); + + kasan_unpoison_shadow(base, watermark - base); } /* |